2024-11-19 17:13:35,638 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-19 17:13:35,653 main DEBUG Took 0.012895 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-19 17:13:35,653 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-19 17:13:35,654 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-19 17:13:35,655 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-19 17:13:35,657 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:13:35,664 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-19 17:13:35,675 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:13:35,677 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:13:35,677 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:13:35,677 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:13:35,678 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:13:35,678 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:13:35,679 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:13:35,679 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:13:35,680 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:13:35,680 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:13:35,681 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:13:35,681 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:13:35,681 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:13:35,682 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:13:35,682 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:13:35,682 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:13:35,683 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:13:35,683 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:13:35,683 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:13:35,684 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:13:35,684 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:13:35,684 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:13:35,685 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:13:35,685 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:13:35,685 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:13:35,686 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-19 17:13:35,687 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:13:35,688 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-19 17:13:35,690 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-19 17:13:35,690 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-19 17:13:35,691 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-19 17:13:35,691 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-19 17:13:35,699 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-19 17:13:35,701 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-19 17:13:35,702 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-19 17:13:35,703 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-19 17:13:35,703 main DEBUG createAppenders(={Console}) 2024-11-19 17:13:35,703 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-19 17:13:35,704 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-19 17:13:35,704 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-19 17:13:35,705 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-19 17:13:35,705 main DEBUG OutputStream closed 2024-11-19 17:13:35,705 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-19 17:13:35,705 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-19 17:13:35,706 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-19 17:13:35,779 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-19 17:13:35,781 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-19 17:13:35,783 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-19 17:13:35,784 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-19 17:13:35,785 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-19 17:13:35,785 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-19 17:13:35,786 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-19 17:13:35,786 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-19 17:13:35,786 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-19 17:13:35,787 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-19 17:13:35,787 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-19 17:13:35,788 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-19 17:13:35,788 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-19 17:13:35,789 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-19 17:13:35,789 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-19 17:13:35,790 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-19 17:13:35,790 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-19 17:13:35,791 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-19 17:13:35,793 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19 17:13:35,794 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-19 17:13:35,794 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-19 17:13:35,795 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-19T17:13:36,031 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e 2024-11-19 17:13:36,034 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-19 17:13:36,035 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19T17:13:36,043 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-19T17:13:36,061 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-19T17:13:36,078 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/cluster_8102c339-c42d-0a52-7e0c-92f0478cebda, deleteOnExit=true 2024-11-19T17:13:36,078 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-19T17:13:36,079 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/test.cache.data in system properties and HBase conf 2024-11-19T17:13:36,079 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/hadoop.tmp.dir in system properties and HBase conf 2024-11-19T17:13:36,080 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/hadoop.log.dir in system properties and HBase conf 2024-11-19T17:13:36,081 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-19T17:13:36,081 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-19T17:13:36,082 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-19T17:13:36,170 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-19T17:13:36,255 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T17:13:36,259 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-19T17:13:36,260 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-19T17:13:36,261 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-19T17:13:36,262 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T17:13:36,262 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-19T17:13:36,263 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-19T17:13:36,263 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T17:13:36,264 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T17:13:36,264 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-19T17:13:36,265 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/nfs.dump.dir in system properties and HBase conf 2024-11-19T17:13:36,266 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/java.io.tmpdir in system properties and HBase conf 2024-11-19T17:13:36,266 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T17:13:36,267 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-19T17:13:36,267 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-19T17:13:38,081 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-19T17:13:38,166 INFO [Time-limited test {}] log.Log(170): Logging initialized @3249ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-19T17:13:38,250 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T17:13:38,316 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T17:13:38,336 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T17:13:38,336 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T17:13:38,337 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-19T17:13:38,350 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T17:13:38,352 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/hadoop.log.dir/,AVAILABLE} 2024-11-19T17:13:38,354 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T17:13:38,552 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/java.io.tmpdir/jetty-localhost-33863-hadoop-hdfs-3_4_1-tests_jar-_-any-4885487464195894645/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-19T17:13:38,559 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:33863} 2024-11-19T17:13:38,560 INFO [Time-limited test {}] server.Server(415): Started @3644ms 2024-11-19T17:13:39,475 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T17:13:39,484 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T17:13:39,485 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T17:13:39,485 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T17:13:39,485 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-19T17:13:39,486 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/hadoop.log.dir/,AVAILABLE} 2024-11-19T17:13:39,487 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T17:13:39,624 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/java.io.tmpdir/jetty-localhost-34989-hadoop-hdfs-3_4_1-tests_jar-_-any-14349145886943415034/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T17:13:39,624 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:34989} 2024-11-19T17:13:39,625 INFO [Time-limited test {}] server.Server(415): Started @4709ms 2024-11-19T17:13:39,690 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-19T17:13:40,587 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/cluster_8102c339-c42d-0a52-7e0c-92f0478cebda/data/data1/current/BP-1636120753-172.17.0.2-1732036416839/current, will proceed with Du for space computation calculation, 2024-11-19T17:13:40,587 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/cluster_8102c339-c42d-0a52-7e0c-92f0478cebda/data/data2/current/BP-1636120753-172.17.0.2-1732036416839/current, will proceed with Du for space computation calculation, 2024-11-19T17:13:40,617 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-19T17:13:40,668 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7cd0ff1e5e6f68f6 with lease ID 0x8da77abbb8ed150d: Processing first storage report for DS-9d3033e5-16c2-469b-8404-5ddbcdf0b7c1 from datanode DatanodeRegistration(127.0.0.1:43609, datanodeUuid=040f1250-2e83-45c5-ad64-23874f827109, infoPort=45389, infoSecurePort=0, ipcPort=41097, storageInfo=lv=-57;cid=testClusterID;nsid=1207866181;c=1732036416839) 2024-11-19T17:13:40,669 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7cd0ff1e5e6f68f6 with lease ID 0x8da77abbb8ed150d: from storage DS-9d3033e5-16c2-469b-8404-5ddbcdf0b7c1 node DatanodeRegistration(127.0.0.1:43609, datanodeUuid=040f1250-2e83-45c5-ad64-23874f827109, infoPort=45389, infoSecurePort=0, ipcPort=41097, storageInfo=lv=-57;cid=testClusterID;nsid=1207866181;c=1732036416839), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-19T17:13:40,669 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7cd0ff1e5e6f68f6 with lease ID 0x8da77abbb8ed150d: Processing first storage report for DS-122cbfb4-cd2f-4830-8859-09e0786d3954 from datanode DatanodeRegistration(127.0.0.1:43609, datanodeUuid=040f1250-2e83-45c5-ad64-23874f827109, infoPort=45389, infoSecurePort=0, ipcPort=41097, storageInfo=lv=-57;cid=testClusterID;nsid=1207866181;c=1732036416839) 2024-11-19T17:13:40,670 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7cd0ff1e5e6f68f6 with lease ID 0x8da77abbb8ed150d: from storage DS-122cbfb4-cd2f-4830-8859-09e0786d3954 node DatanodeRegistration(127.0.0.1:43609, datanodeUuid=040f1250-2e83-45c5-ad64-23874f827109, infoPort=45389, infoSecurePort=0, ipcPort=41097, storageInfo=lv=-57;cid=testClusterID;nsid=1207866181;c=1732036416839), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-19T17:13:40,713 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e 2024-11-19T17:13:40,801 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/cluster_8102c339-c42d-0a52-7e0c-92f0478cebda/zookeeper_0, clientPort=56338, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/cluster_8102c339-c42d-0a52-7e0c-92f0478cebda/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/cluster_8102c339-c42d-0a52-7e0c-92f0478cebda/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-19T17:13:40,812 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56338 2024-11-19T17:13:40,827 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T17:13:40,832 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T17:13:41,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741825_1001 (size=7) 2024-11-19T17:13:41,496 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e with version=8 2024-11-19T17:13:41,497 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/hbase-staging 2024-11-19T17:13:41,639 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-19T17:13:41,911 INFO [Time-limited test {}] client.ConnectionUtils(128): master/280a42d09ec8:0 server-side Connection retries=45 2024-11-19T17:13:41,921 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T17:13:41,922 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T17:13:41,926 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T17:13:41,926 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T17:13:41,927 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T17:13:42,062 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-19T17:13:42,137 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-19T17:13:42,146 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-19T17:13:42,151 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T17:13:42,179 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3066 (auto-detected) 2024-11-19T17:13:42,181 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-19T17:13:42,207 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40603 2024-11-19T17:13:42,235 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40603 connecting to ZooKeeper ensemble=127.0.0.1:56338 2024-11-19T17:13:42,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:406030x0, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T17:13:42,281 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40603-0x10131384f3a0000 connected 2024-11-19T17:13:42,318 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T17:13:42,320 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T17:13:42,340 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40603-0x10131384f3a0000, quorum=127.0.0.1:56338, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T17:13:42,345 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e, hbase.cluster.distributed=false 2024-11-19T17:13:42,370 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40603-0x10131384f3a0000, quorum=127.0.0.1:56338, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T17:13:42,375 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40603 2024-11-19T17:13:42,376 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40603 2024-11-19T17:13:42,379 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40603 2024-11-19T17:13:42,380 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40603 2024-11-19T17:13:42,381 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40603 2024-11-19T17:13:42,512 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/280a42d09ec8:0 server-side Connection retries=45 2024-11-19T17:13:42,514 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T17:13:42,514 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T17:13:42,514 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T17:13:42,514 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T17:13:42,514 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T17:13:42,517 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T17:13:42,519 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T17:13:42,520 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33873 2024-11-19T17:13:42,522 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33873 connecting to ZooKeeper ensemble=127.0.0.1:56338 2024-11-19T17:13:42,523 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T17:13:42,526 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T17:13:42,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:338730x0, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T17:13:42,535 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33873-0x10131384f3a0001 connected 2024-11-19T17:13:42,535 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33873-0x10131384f3a0001, quorum=127.0.0.1:56338, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T17:13:42,540 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T17:13:42,547 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T17:13:42,550 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33873-0x10131384f3a0001, quorum=127.0.0.1:56338, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T17:13:42,554 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33873-0x10131384f3a0001, quorum=127.0.0.1:56338, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T17:13:42,555 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33873 2024-11-19T17:13:42,555 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33873 2024-11-19T17:13:42,556 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33873 2024-11-19T17:13:42,556 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33873 2024-11-19T17:13:42,557 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33873 2024-11-19T17:13:42,573 DEBUG [M:0;280a42d09ec8:40603 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;280a42d09ec8:40603 2024-11-19T17:13:42,574 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/280a42d09ec8,40603,1732036421715 2024-11-19T17:13:42,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33873-0x10131384f3a0001, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T17:13:42,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x10131384f3a0000, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T17:13:42,583 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40603-0x10131384f3a0000, quorum=127.0.0.1:56338, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/280a42d09ec8,40603,1732036421715 2024-11-19T17:13:42,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33873-0x10131384f3a0001, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T17:13:42,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x10131384f3a0000, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:13:42,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33873-0x10131384f3a0001, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:13:42,604 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40603-0x10131384f3a0000, quorum=127.0.0.1:56338, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-19T17:13:42,605 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/280a42d09ec8,40603,1732036421715 from backup master directory 2024-11-19T17:13:42,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x10131384f3a0000, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/280a42d09ec8,40603,1732036421715 2024-11-19T17:13:42,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33873-0x10131384f3a0001, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T17:13:42,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x10131384f3a0000, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T17:13:42,608 WARN [master/280a42d09ec8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T17:13:42,608 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=280a42d09ec8,40603,1732036421715 2024-11-19T17:13:42,611 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-19T17:13:42,613 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-19T17:13:42,670 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/hbase.id] with ID: e13b793f-957c-4380-9440-a4e203c4ee51 2024-11-19T17:13:42,670 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/.tmp/hbase.id 2024-11-19T17:13:42,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741826_1002 (size=42) 2024-11-19T17:13:43,082 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/.tmp/hbase.id]:[hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/hbase.id] 2024-11-19T17:13:43,123 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T17:13:43,127 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-19T17:13:43,146 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-19T17:13:43,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x10131384f3a0000, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:13:43,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33873-0x10131384f3a0001, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:13:43,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741827_1003 (size=196) 2024-11-19T17:13:43,586 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T17:13:43,589 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-19T17:13:43,606 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:43,610 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T17:13:43,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741828_1004 (size=1189) 2024-11-19T17:13:44,062 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/MasterData/data/master/store 2024-11-19T17:13:44,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741829_1005 (size=34) 2024-11-19T17:13:44,487 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-19T17:13:44,490 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:13:44,491 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T17:13:44,491 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:13:44,492 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:13:44,493 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T17:13:44,493 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:13:44,493 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:13:44,494 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732036424491Disabling compacts and flushes for region at 1732036424491Disabling writes for close at 1732036424493 (+2 ms)Writing region close event to WAL at 1732036424493Closed at 1732036424493 2024-11-19T17:13:44,497 WARN [master/280a42d09ec8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/MasterData/data/master/store/.initializing 2024-11-19T17:13:44,497 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/MasterData/WALs/280a42d09ec8,40603,1732036421715 2024-11-19T17:13:44,505 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T17:13:44,520 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=280a42d09ec8%2C40603%2C1732036421715, suffix=, logDir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/MasterData/WALs/280a42d09ec8,40603,1732036421715, archiveDir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/MasterData/oldWALs, maxLogs=10 2024-11-19T17:13:44,543 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/MasterData/WALs/280a42d09ec8,40603,1732036421715/280a42d09ec8%2C40603%2C1732036421715.1732036424524, exclude list is [], retry=0 2024-11-19T17:13:44,560 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43609,DS-9d3033e5-16c2-469b-8404-5ddbcdf0b7c1,DISK] 2024-11-19T17:13:44,563 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-19T17:13:44,599 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/MasterData/WALs/280a42d09ec8,40603,1732036421715/280a42d09ec8%2C40603%2C1732036421715.1732036424524 2024-11-19T17:13:44,600 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45389:45389)] 2024-11-19T17:13:44,600 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:13:44,601 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:13:44,604 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:13:44,605 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:13:44,642 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:13:44,668 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-19T17:13:44,672 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:13:44,674 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:13:44,675 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:13:44,678 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-19T17:13:44,678 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:13:44,679 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:13:44,680 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:13:44,682 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-19T17:13:44,683 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:13:44,683 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:13:44,684 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:13:44,686 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-19T17:13:44,686 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:13:44,687 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:13:44,688 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:13:44,691 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:13:44,692 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:13:44,698 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:13:44,698 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:13:44,702 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T17:13:44,706 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:13:44,716 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T17:13:44,733 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73710254, jitterRate=0.09836837649345398}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T17:13:44,741 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732036424616Initializing all the Stores at 1732036424619 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732036424619Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036424620 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036424620Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036424620Cleaning up temporary data from old regions at 1732036424698 (+78 ms)Region opened successfully at 1732036424741 (+43 ms) 2024-11-19T17:13:44,743 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-19T17:13:44,790 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7666383, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=280a42d09ec8/172.17.0.2:0 2024-11-19T17:13:44,832 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-19T17:13:44,845 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-19T17:13:44,846 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-19T17:13:44,849 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T17:13:44,851 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-19T17:13:44,857 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-19T17:13:44,857 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-19T17:13:44,885 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-19T17:13:44,894 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40603-0x10131384f3a0000, quorum=127.0.0.1:56338, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-19T17:13:44,897 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-19T17:13:44,899 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-19T17:13:44,901 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40603-0x10131384f3a0000, quorum=127.0.0.1:56338, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-19T17:13:44,903 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-19T17:13:44,905 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-19T17:13:44,909 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40603-0x10131384f3a0000, quorum=127.0.0.1:56338, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-19T17:13:44,910 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-19T17:13:44,912 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40603-0x10131384f3a0000, quorum=127.0.0.1:56338, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-19T17:13:44,913 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-19T17:13:44,932 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40603-0x10131384f3a0000, quorum=127.0.0.1:56338, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-19T17:13:44,934 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-19T17:13:44,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33873-0x10131384f3a0001, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T17:13:44,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x10131384f3a0000, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T17:13:44,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33873-0x10131384f3a0001, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:13:44,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x10131384f3a0000, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:13:44,947 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=280a42d09ec8,40603,1732036421715, sessionid=0x10131384f3a0000, setting cluster-up flag (Was=false) 2024-11-19T17:13:44,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x10131384f3a0000, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:13:44,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33873-0x10131384f3a0001, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:13:44,968 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-19T17:13:44,970 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=280a42d09ec8,40603,1732036421715 2024-11-19T17:13:44,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33873-0x10131384f3a0001, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:13:44,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x10131384f3a0000, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:13:44,982 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-19T17:13:44,983 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=280a42d09ec8,40603,1732036421715 2024-11-19T17:13:44,990 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-19T17:13:45,062 INFO [RS:0;280a42d09ec8:33873 {}] regionserver.HRegionServer(746): ClusterId : e13b793f-957c-4380-9440-a4e203c4ee51 2024-11-19T17:13:45,065 DEBUG [RS:0;280a42d09ec8:33873 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T17:13:45,068 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T17:13:45,071 DEBUG [RS:0;280a42d09ec8:33873 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T17:13:45,071 DEBUG [RS:0;280a42d09ec8:33873 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T17:13:45,077 DEBUG [RS:0;280a42d09ec8:33873 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T17:13:45,078 DEBUG [RS:0;280a42d09ec8:33873 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@247909d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=280a42d09ec8/172.17.0.2:0 2024-11-19T17:13:45,079 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-19T17:13:45,086 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-19T17:13:45,092 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 280a42d09ec8,40603,1732036421715 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-19T17:13:45,096 DEBUG [RS:0;280a42d09ec8:33873 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;280a42d09ec8:33873 2024-11-19T17:13:45,099 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/280a42d09ec8:0, corePoolSize=5, maxPoolSize=5 2024-11-19T17:13:45,099 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/280a42d09ec8:0, corePoolSize=5, maxPoolSize=5 2024-11-19T17:13:45,099 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/280a42d09ec8:0, corePoolSize=5, maxPoolSize=5 2024-11-19T17:13:45,099 INFO [RS:0;280a42d09ec8:33873 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T17:13:45,099 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/280a42d09ec8:0, corePoolSize=5, maxPoolSize=5 2024-11-19T17:13:45,099 INFO [RS:0;280a42d09ec8:33873 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T17:13:45,099 DEBUG [RS:0;280a42d09ec8:33873 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T17:13:45,100 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/280a42d09ec8:0, corePoolSize=10, maxPoolSize=10 2024-11-19T17:13:45,100 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/280a42d09ec8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:13:45,100 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/280a42d09ec8:0, corePoolSize=2, maxPoolSize=2 2024-11-19T17:13:45,100 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/280a42d09ec8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:13:45,102 INFO [RS:0;280a42d09ec8:33873 {}] regionserver.HRegionServer(2659): reportForDuty to master=280a42d09ec8,40603,1732036421715 with port=33873, startcode=1732036422473 2024-11-19T17:13:45,103 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732036455102 2024-11-19T17:13:45,104 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-19T17:13:45,105 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-19T17:13:45,106 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T17:13:45,107 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-19T17:13:45,109 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-19T17:13:45,110 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-19T17:13:45,110 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-19T17:13:45,110 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-19T17:13:45,111 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T17:13:45,115 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:13:45,116 DEBUG [RS:0;280a42d09ec8:33873 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T17:13:45,115 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-19T17:13:45,117 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-19T17:13:45,118 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-19T17:13:45,119 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-19T17:13:45,121 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-19T17:13:45,122 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-19T17:13:45,124 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/280a42d09ec8:0:becomeActiveMaster-HFileCleaner.large.0-1732036425123,5,FailOnTimeoutGroup] 2024-11-19T17:13:45,125 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/280a42d09ec8:0:becomeActiveMaster-HFileCleaner.small.0-1732036425124,5,FailOnTimeoutGroup] 2024-11-19T17:13:45,125 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T17:13:45,125 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-19T17:13:45,127 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-19T17:13:45,127 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-19T17:13:45,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741831_1007 (size=1321) 2024-11-19T17:13:45,161 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42695, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T17:13:45,167 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40603 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 280a42d09ec8,33873,1732036422473 2024-11-19T17:13:45,170 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40603 {}] master.ServerManager(517): Registering regionserver=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:45,184 DEBUG [RS:0;280a42d09ec8:33873 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e 2024-11-19T17:13:45,185 DEBUG [RS:0;280a42d09ec8:33873 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34575 2024-11-19T17:13:45,185 DEBUG [RS:0;280a42d09ec8:33873 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T17:13:45,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x10131384f3a0000, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T17:13:45,190 DEBUG [RS:0;280a42d09ec8:33873 {}] zookeeper.ZKUtil(111): regionserver:33873-0x10131384f3a0001, quorum=127.0.0.1:56338, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/280a42d09ec8,33873,1732036422473 2024-11-19T17:13:45,190 WARN [RS:0;280a42d09ec8:33873 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T17:13:45,190 INFO [RS:0;280a42d09ec8:33873 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T17:13:45,190 DEBUG [RS:0;280a42d09ec8:33873 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/WALs/280a42d09ec8,33873,1732036422473 2024-11-19T17:13:45,192 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [280a42d09ec8,33873,1732036422473] 2024-11-19T17:13:45,219 INFO [RS:0;280a42d09ec8:33873 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T17:13:45,231 INFO [RS:0;280a42d09ec8:33873 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T17:13:45,236 INFO [RS:0;280a42d09ec8:33873 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T17:13:45,236 INFO [RS:0;280a42d09ec8:33873 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T17:13:45,237 INFO [RS:0;280a42d09ec8:33873 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T17:13:45,243 INFO [RS:0;280a42d09ec8:33873 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T17:13:45,244 INFO [RS:0;280a42d09ec8:33873 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T17:13:45,244 DEBUG [RS:0;280a42d09ec8:33873 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/280a42d09ec8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:13:45,244 DEBUG [RS:0;280a42d09ec8:33873 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/280a42d09ec8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:13:45,245 DEBUG [RS:0;280a42d09ec8:33873 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/280a42d09ec8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:13:45,245 DEBUG [RS:0;280a42d09ec8:33873 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/280a42d09ec8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:13:45,245 DEBUG [RS:0;280a42d09ec8:33873 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/280a42d09ec8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:13:45,245 DEBUG [RS:0;280a42d09ec8:33873 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/280a42d09ec8:0, corePoolSize=2, maxPoolSize=2 2024-11-19T17:13:45,245 DEBUG [RS:0;280a42d09ec8:33873 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/280a42d09ec8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:13:45,245 DEBUG [RS:0;280a42d09ec8:33873 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/280a42d09ec8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:13:45,245 DEBUG [RS:0;280a42d09ec8:33873 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/280a42d09ec8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:13:45,246 DEBUG [RS:0;280a42d09ec8:33873 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/280a42d09ec8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:13:45,246 DEBUG [RS:0;280a42d09ec8:33873 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/280a42d09ec8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:13:45,246 DEBUG [RS:0;280a42d09ec8:33873 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/280a42d09ec8:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:13:45,246 DEBUG [RS:0;280a42d09ec8:33873 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/280a42d09ec8:0, corePoolSize=3, maxPoolSize=3 2024-11-19T17:13:45,246 DEBUG [RS:0;280a42d09ec8:33873 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0, corePoolSize=3, maxPoolSize=3 2024-11-19T17:13:45,248 INFO [RS:0;280a42d09ec8:33873 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T17:13:45,248 INFO [RS:0;280a42d09ec8:33873 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T17:13:45,248 INFO [RS:0;280a42d09ec8:33873 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T17:13:45,248 INFO [RS:0;280a42d09ec8:33873 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T17:13:45,248 INFO [RS:0;280a42d09ec8:33873 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T17:13:45,249 INFO [RS:0;280a42d09ec8:33873 {}] hbase.ChoreService(168): Chore ScheduledChore name=280a42d09ec8,33873,1732036422473-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T17:13:45,269 INFO [RS:0;280a42d09ec8:33873 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T17:13:45,271 INFO [RS:0;280a42d09ec8:33873 {}] hbase.ChoreService(168): Chore ScheduledChore name=280a42d09ec8,33873,1732036422473-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T17:13:45,272 INFO [RS:0;280a42d09ec8:33873 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T17:13:45,272 INFO [RS:0;280a42d09ec8:33873 {}] regionserver.Replication(171): 280a42d09ec8,33873,1732036422473 started 2024-11-19T17:13:45,297 INFO [RS:0;280a42d09ec8:33873 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T17:13:45,297 INFO [RS:0;280a42d09ec8:33873 {}] regionserver.HRegionServer(1482): Serving as 280a42d09ec8,33873,1732036422473, RpcServer on 280a42d09ec8/172.17.0.2:33873, sessionid=0x10131384f3a0001 2024-11-19T17:13:45,298 DEBUG [RS:0;280a42d09ec8:33873 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T17:13:45,298 DEBUG [RS:0;280a42d09ec8:33873 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 280a42d09ec8,33873,1732036422473 2024-11-19T17:13:45,299 DEBUG [RS:0;280a42d09ec8:33873 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '280a42d09ec8,33873,1732036422473' 2024-11-19T17:13:45,299 DEBUG [RS:0;280a42d09ec8:33873 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T17:13:45,300 DEBUG [RS:0;280a42d09ec8:33873 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T17:13:45,301 DEBUG [RS:0;280a42d09ec8:33873 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T17:13:45,301 DEBUG [RS:0;280a42d09ec8:33873 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T17:13:45,301 DEBUG [RS:0;280a42d09ec8:33873 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 280a42d09ec8,33873,1732036422473 2024-11-19T17:13:45,302 DEBUG [RS:0;280a42d09ec8:33873 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '280a42d09ec8,33873,1732036422473' 2024-11-19T17:13:45,302 DEBUG [RS:0;280a42d09ec8:33873 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T17:13:45,303 DEBUG [RS:0;280a42d09ec8:33873 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T17:13:45,304 DEBUG [RS:0;280a42d09ec8:33873 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T17:13:45,304 INFO [RS:0;280a42d09ec8:33873 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T17:13:45,304 INFO [RS:0;280a42d09ec8:33873 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T17:13:45,411 INFO [RS:0;280a42d09ec8:33873 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T17:13:45,415 INFO [RS:0;280a42d09ec8:33873 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=280a42d09ec8%2C33873%2C1732036422473, suffix=, logDir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/WALs/280a42d09ec8,33873,1732036422473, archiveDir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/oldWALs, maxLogs=32 2024-11-19T17:13:45,434 DEBUG [RS:0;280a42d09ec8:33873 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/WALs/280a42d09ec8,33873,1732036422473/280a42d09ec8%2C33873%2C1732036422473.1732036425418, exclude list is [], retry=0 2024-11-19T17:13:45,440 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43609,DS-9d3033e5-16c2-469b-8404-5ddbcdf0b7c1,DISK] 2024-11-19T17:13:45,445 INFO [RS:0;280a42d09ec8:33873 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/WALs/280a42d09ec8,33873,1732036422473/280a42d09ec8%2C33873%2C1732036422473.1732036425418 2024-11-19T17:13:45,445 DEBUG [RS:0;280a42d09ec8:33873 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45389:45389)] 2024-11-19T17:13:45,537 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-19T17:13:45,537 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e 2024-11-19T17:13:45,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741833_1009 (size=32) 2024-11-19T17:13:45,554 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:13:45,562 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T17:13:45,565 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T17:13:45,566 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:13:45,567 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:13:45,567 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T17:13:45,570 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T17:13:45,571 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:13:45,572 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:13:45,572 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T17:13:45,575 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T17:13:45,575 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:13:45,576 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:13:45,576 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T17:13:45,579 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T17:13:45,580 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:13:45,581 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:13:45,581 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T17:13:45,583 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/hbase/meta/1588230740 2024-11-19T17:13:45,584 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/hbase/meta/1588230740 2024-11-19T17:13:45,589 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T17:13:45,589 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T17:13:45,591 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T17:13:45,593 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T17:13:45,599 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T17:13:45,600 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68073352, jitterRate=0.014371991157531738}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T17:13:45,604 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732036425554Initializing all the Stores at 1732036425556 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732036425556Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732036425562 (+6 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036425562Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732036425562Cleaning up temporary data from old regions at 1732036425589 (+27 ms)Region opened successfully at 1732036425603 (+14 ms) 2024-11-19T17:13:45,604 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T17:13:45,604 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T17:13:45,604 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T17:13:45,604 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T17:13:45,605 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T17:13:45,607 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T17:13:45,607 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732036425604Disabling compacts and flushes for region at 1732036425604Disabling writes for close at 1732036425604Writing region close event to WAL at 1732036425606 (+2 ms)Closed at 1732036425607 (+1 ms) 2024-11-19T17:13:45,611 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T17:13:45,612 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-19T17:13:45,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-19T17:13:45,630 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T17:13:45,633 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-19T17:13:45,787 DEBUG [280a42d09ec8:40603 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-19T17:13:45,801 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:45,808 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 280a42d09ec8,33873,1732036422473, state=OPENING 2024-11-19T17:13:45,816 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-19T17:13:45,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33873-0x10131384f3a0001, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:13:45,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x10131384f3a0000, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:13:45,819 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T17:13:45,820 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T17:13:45,821 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T17:13:45,823 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=280a42d09ec8,33873,1732036422473}] 2024-11-19T17:13:46,002 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T17:13:46,006 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51433, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T17:13:46,017 INFO [RS_OPEN_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-19T17:13:46,018 INFO [RS_OPEN_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T17:13:46,018 INFO [RS_OPEN_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-19T17:13:46,022 INFO [RS_OPEN_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=280a42d09ec8%2C33873%2C1732036422473.meta, suffix=.meta, logDir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/WALs/280a42d09ec8,33873,1732036422473, archiveDir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/oldWALs, maxLogs=32 2024-11-19T17:13:46,040 DEBUG [RS_OPEN_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/WALs/280a42d09ec8,33873,1732036422473/280a42d09ec8%2C33873%2C1732036422473.meta.1732036426024.meta, exclude list is [], retry=0 2024-11-19T17:13:46,045 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43609,DS-9d3033e5-16c2-469b-8404-5ddbcdf0b7c1,DISK] 2024-11-19T17:13:46,048 INFO [RS_OPEN_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/WALs/280a42d09ec8,33873,1732036422473/280a42d09ec8%2C33873%2C1732036422473.meta.1732036426024.meta 2024-11-19T17:13:46,049 DEBUG [RS_OPEN_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45389:45389)] 2024-11-19T17:13:46,050 DEBUG [RS_OPEN_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:13:46,052 DEBUG [RS_OPEN_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-19T17:13:46,056 DEBUG [RS_OPEN_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-19T17:13:46,062 INFO [RS_OPEN_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-19T17:13:46,068 DEBUG [RS_OPEN_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-19T17:13:46,068 DEBUG [RS_OPEN_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:13:46,069 DEBUG [RS_OPEN_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-19T17:13:46,069 DEBUG [RS_OPEN_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-19T17:13:46,072 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T17:13:46,074 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T17:13:46,074 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:13:46,075 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:13:46,075 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T17:13:46,077 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T17:13:46,077 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:13:46,078 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:13:46,078 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T17:13:46,080 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T17:13:46,080 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:13:46,081 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:13:46,081 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T17:13:46,083 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T17:13:46,083 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:13:46,084 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:13:46,084 DEBUG [RS_OPEN_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T17:13:46,086 DEBUG [RS_OPEN_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/hbase/meta/1588230740 2024-11-19T17:13:46,089 DEBUG [RS_OPEN_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/hbase/meta/1588230740 2024-11-19T17:13:46,092 DEBUG [RS_OPEN_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T17:13:46,092 DEBUG [RS_OPEN_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T17:13:46,093 DEBUG [RS_OPEN_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T17:13:46,097 DEBUG [RS_OPEN_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T17:13:46,099 INFO [RS_OPEN_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68378798, jitterRate=0.01892349123954773}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T17:13:46,100 DEBUG [RS_OPEN_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-19T17:13:46,102 DEBUG [RS_OPEN_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732036426069Writing region info on filesystem at 1732036426070 (+1 ms)Initializing all the Stores at 1732036426072 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732036426072Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732036426072Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036426072Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732036426072Cleaning up temporary data from old regions at 1732036426092 (+20 ms)Running coprocessor post-open hooks at 1732036426100 (+8 ms)Region opened successfully at 1732036426102 (+2 ms) 2024-11-19T17:13:46,111 INFO [RS_OPEN_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732036425992 2024-11-19T17:13:46,125 DEBUG [RS_OPEN_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-19T17:13:46,126 INFO [RS_OPEN_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-19T17:13:46,127 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:46,130 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 280a42d09ec8,33873,1732036422473, state=OPEN 2024-11-19T17:13:46,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x10131384f3a0000, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T17:13:46,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33873-0x10131384f3a0001, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T17:13:46,137 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T17:13:46,138 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:46,138 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T17:13:46,144 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-19T17:13:46,145 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=280a42d09ec8,33873,1732036422473 in 315 msec 2024-11-19T17:13:46,151 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-19T17:13:46,151 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 527 msec 2024-11-19T17:13:46,152 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T17:13:46,153 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-19T17:13:46,174 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:13:46,175 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:13:46,197 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:13:46,200 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39003, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:13:46,221 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2000 sec 2024-11-19T17:13:46,221 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732036426221, completionTime=-1 2024-11-19T17:13:46,224 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-19T17:13:46,224 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T17:13:46,252 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-19T17:13:46,252 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732036486252 2024-11-19T17:13:46,252 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732036546252 2024-11-19T17:13:46,252 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 28 msec 2024-11-19T17:13:46,254 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=280a42d09ec8,40603,1732036421715-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T17:13:46,255 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=280a42d09ec8,40603,1732036421715-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T17:13:46,255 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=280a42d09ec8,40603,1732036421715-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T17:13:46,257 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-280a42d09ec8:40603, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T17:13:46,257 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-19T17:13:46,258 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-19T17:13:46,265 DEBUG [master/280a42d09ec8:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-19T17:13:46,314 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.705sec 2024-11-19T17:13:46,315 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-19T17:13:46,316 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-19T17:13:46,321 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-19T17:13:46,321 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-19T17:13:46,321 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-19T17:13:46,322 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=280a42d09ec8,40603,1732036421715-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T17:13:46,323 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=280a42d09ec8,40603,1732036421715-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-19T17:13:46,331 DEBUG [master/280a42d09ec8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-19T17:13:46,332 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-19T17:13:46,333 INFO [master/280a42d09ec8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=280a42d09ec8,40603,1732036421715-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T17:13:46,372 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45544f78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:46,374 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-19T17:13:46,374 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-19T17:13:46,377 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:13:46,380 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:13:46,387 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:13:46,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:13:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:13:46,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@424b8b64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:46,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:13:46,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:13:46,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:13:46,397 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:13:46,400 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2de67725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:46,400 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:13:46,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:13:46,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:13:46,410 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:13:46,430 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=280a42d09ec8,40603,1732036421715 2024-11-19T17:13:46,442 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=317, ProcessCount=11, AvailableMemoryMB=7196 2024-11-19T17:13:46,450 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T17:13:46,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 280a42d09ec8,40603,1732036421715 2024-11-19T17:13:46,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@27423f7d 2024-11-19T17:13:46,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T17:13:46,461 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T17:13:46,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T17:13:46,467 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T17:13:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T17:13:46,477 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T17:13:46,479 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-19T17:13:46,479 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:13:46,481 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T17:13:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T17:13:46,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741835_1011 (size=1061) 2024-11-19T17:13:46,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T17:13:46,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T17:13:46,897 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e 2024-11-19T17:13:46,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741836_1012 (size=53) 2024-11-19T17:13:47,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T17:13:47,309 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:13:47,309 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 689491498ff99efc87002cc59bc70ef3, disabling compactions & flushes 2024-11-19T17:13:47,309 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:47,309 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:47,309 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. after waiting 0 ms 2024-11-19T17:13:47,309 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:47,309 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:47,309 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 689491498ff99efc87002cc59bc70ef3: Waiting for close lock at 1732036427309Disabling compacts and flushes for region at 1732036427309Disabling writes for close at 1732036427309Writing region close event to WAL at 1732036427309Closed at 1732036427309 2024-11-19T17:13:47,312 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T17:13:47,317 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732036427312"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732036427312"}]},"ts":"1732036427312"} 2024-11-19T17:13:47,322 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T17:13:47,324 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T17:13:47,327 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732036427324"}]},"ts":"1732036427324"} 2024-11-19T17:13:47,331 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T17:13:47,333 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=689491498ff99efc87002cc59bc70ef3, ASSIGN}] 2024-11-19T17:13:47,335 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=689491498ff99efc87002cc59bc70ef3, ASSIGN 2024-11-19T17:13:47,337 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=689491498ff99efc87002cc59bc70ef3, ASSIGN; state=OFFLINE, location=280a42d09ec8,33873,1732036422473; forceNewPlan=false, retain=false 2024-11-19T17:13:47,489 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=689491498ff99efc87002cc59bc70ef3, regionState=OPENING, regionLocation=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:47,493 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=689491498ff99efc87002cc59bc70ef3, ASSIGN because future has completed 2024-11-19T17:13:47,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473}] 2024-11-19T17:13:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T17:13:47,655 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:47,656 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:13:47,656 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:47,656 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:13:47,657 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:47,657 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:47,659 INFO [StoreOpener-689491498ff99efc87002cc59bc70ef3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:47,662 INFO [StoreOpener-689491498ff99efc87002cc59bc70ef3-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T17:13:47,663 INFO [StoreOpener-689491498ff99efc87002cc59bc70ef3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 689491498ff99efc87002cc59bc70ef3 columnFamilyName A 2024-11-19T17:13:47,663 DEBUG [StoreOpener-689491498ff99efc87002cc59bc70ef3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:13:47,664 INFO [StoreOpener-689491498ff99efc87002cc59bc70ef3-1 {}] regionserver.HStore(327): Store=689491498ff99efc87002cc59bc70ef3/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:13:47,664 INFO [StoreOpener-689491498ff99efc87002cc59bc70ef3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:47,666 INFO [StoreOpener-689491498ff99efc87002cc59bc70ef3-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T17:13:47,667 INFO [StoreOpener-689491498ff99efc87002cc59bc70ef3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 689491498ff99efc87002cc59bc70ef3 columnFamilyName B 2024-11-19T17:13:47,667 DEBUG [StoreOpener-689491498ff99efc87002cc59bc70ef3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:13:47,668 INFO [StoreOpener-689491498ff99efc87002cc59bc70ef3-1 {}] regionserver.HStore(327): Store=689491498ff99efc87002cc59bc70ef3/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:13:47,668 INFO [StoreOpener-689491498ff99efc87002cc59bc70ef3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:47,670 INFO [StoreOpener-689491498ff99efc87002cc59bc70ef3-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T17:13:47,670 INFO [StoreOpener-689491498ff99efc87002cc59bc70ef3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 689491498ff99efc87002cc59bc70ef3 columnFamilyName C 2024-11-19T17:13:47,670 DEBUG [StoreOpener-689491498ff99efc87002cc59bc70ef3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:13:47,671 INFO [StoreOpener-689491498ff99efc87002cc59bc70ef3-1 {}] regionserver.HStore(327): Store=689491498ff99efc87002cc59bc70ef3/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:13:47,671 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:47,672 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:47,673 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:47,674 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:47,676 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:47,676 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:47,678 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T17:13:47,680 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:47,684 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T17:13:47,685 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 689491498ff99efc87002cc59bc70ef3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60602553, jitterRate=-0.09695158898830414}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T17:13:47,685 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:47,686 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 689491498ff99efc87002cc59bc70ef3: Running coprocessor pre-open hook at 1732036427657Writing region info on filesystem at 1732036427657Initializing all the Stores at 1732036427659 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036427659Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036427659Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036427659Cleaning up temporary data from old regions at 1732036427677 (+18 ms)Running coprocessor post-open hooks at 1732036427685 (+8 ms)Region opened successfully at 1732036427686 (+1 ms) 2024-11-19T17:13:47,688 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., pid=6, masterSystemTime=1732036427648 2024-11-19T17:13:47,692 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:47,692 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:47,693 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=689491498ff99efc87002cc59bc70ef3, regionState=OPEN, openSeqNum=2, regionLocation=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:47,697 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 because future has completed 2024-11-19T17:13:47,703 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-19T17:13:47,703 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 in 205 msec 2024-11-19T17:13:47,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-19T17:13:47,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=689491498ff99efc87002cc59bc70ef3, ASSIGN in 370 msec 2024-11-19T17:13:47,709 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T17:13:47,709 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732036427709"}]},"ts":"1732036427709"} 2024-11-19T17:13:47,712 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T17:13:47,713 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T17:13:47,716 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2430 sec 2024-11-19T17:13:48,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T17:13:48,646 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:13:48,651 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19b2c994, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:48,651 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:13:48,652 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:13:48,653 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:13:48,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:13:48,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:13:48,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a4b96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:48,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:13:48,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:13:48,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:13:48,656 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:13:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@185f36b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:48,658 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:13:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:13:48,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:13:48,662 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:13:48,664 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T17:13:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 280a42d09ec8,40603,1732036421715 2024-11-19T17:13:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@25ca1f99 2024-11-19T17:13:48,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T17:13:48,670 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T17:13:48,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a52ffa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:48,674 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:13:48,674 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:13:48,675 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:13:48,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:13:48,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:13:48,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68cf4e74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:48,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:13:48,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:13:48,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:13:48,679 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:13:48,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9a6e4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:48,680 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f14cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:48,680 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:13:48,681 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:13:48,682 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:13:48,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:13:48,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:13:48,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@89c0433, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:48,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:13:48,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:13:48,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:13:48,685 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:13:48,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ef39c97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:48,686 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:48,686 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:13:48,687 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:13:48,688 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:13:48,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:13:48,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:13:48,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8ac8568, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:48,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:13:48,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:13:48,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:13:48,692 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:13:48,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ab1aa4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:48,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc41cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:48,694 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:13:48,694 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:13:48,696 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:13:48,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:13:48,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:13:48,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f9668b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:48,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:13:48,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:13:48,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:13:48,699 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:13:48,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c1e7e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:48,700 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2900b22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:48,700 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:13:48,701 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:13:48,701 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:13:48,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:13:48,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:13:48,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b21b95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:48,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:13:48,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:13:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:13:48,703 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:13:48,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a8d6bd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:48,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d39686f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:48,705 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:13:48,706 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:13:48,707 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:13:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:13:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:13:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bc03052, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:13:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:13:48,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:13:48,709 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:13:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267419b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:48,711 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6963d59b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:48,711 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:13:48,711 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:13:48,712 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:13:48,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:13:48,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:13:48,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a88f812, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:48,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:13:48,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:13:48,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:13:48,714 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:13:48,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c7f9af0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:48,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9a31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:48,716 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:13:48,717 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:13:48,717 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:13:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:13:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:13:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53def629, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:13:48,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:13:48,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:13:48,720 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:13:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e3080a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:48,722 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0b495b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:48,722 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:13:48,722 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:13:48,723 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:13:48,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:13:48,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:13:48,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c6ea95c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:48,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:13:48,725 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:13:48,725 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:13:48,726 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:13:48,727 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a0f145b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:13:48,730 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:13:48,730 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:13:48,731 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:13:48,730 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:13:48,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:13:48,732 DEBUG [Thread-158 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:13:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:13:48,733 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:13:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:13:48,733 DEBUG [Thread-158 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:13:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:13:48,734 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:13:48,735 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:13:48,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:13:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:13:48,736 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:13:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:13:48,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:13:48,737 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:13:48,737 DEBUG [Thread-160 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:13:48,738 DEBUG [Thread-160 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:13:48,738 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:13:48,739 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:13:48,740 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:13:48,743 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:13:48,745 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:13:48,745 DEBUG [Thread-156 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:13:48,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2] 2024-11-19T17:13:48,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2] 2024-11-19T17:13:48,746 DEBUG [Thread-156 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:13:48,747 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:13:48,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2] 2024-11-19T17:13:48,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2] 2024-11-19T17:13:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2] 2024-11-19T17:13:48,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2] 2024-11-19T17:13:48,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2] 2024-11-19T17:13:48,759 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:13:48,759 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:13:48,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-19T17:13:48,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:13:48,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:13:48,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:13:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:13:48,763 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:13:48,763 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:13:48,763 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:13:48,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T17:13:48,767 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:13:48,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2] 2024-11-19T17:13:48,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2] 2024-11-19T17:13:48,769 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:13:48,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:48,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T17:13:48,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:13:48,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:48,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:13:48,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:48,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:13:48,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:48,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T17:13:48,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T17:13:48,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:48,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:48,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:48,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:48,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:48,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/412a1bba31264b349a9732ae7b8883d4 is 50, key is test_row_0/A:col10/1732036428817/Put/seqid=0 2024-11-19T17:13:48,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:49,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741837_1013 (size=12001) 2024-11-19T17:13:49,015 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036428957, max timestamp=1732036429003Num uniques 859; 2024-11-19T17:13:49,017 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:13:49,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036428957, max timestamp=1732036429007Num uniques 859; 2024-11-19T17:13:49,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:13:49,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036428957, max timestamp=1732036429003Num uniques 859; 2024-11-19T17:13:49,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:13:49,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T17:13:49,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:49,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:49,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036489080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:49,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036489075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:49,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:49,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036489083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:49,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:49,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036489094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:49,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:49,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036489099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:49,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T17:13:49,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:49,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:49,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:49,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:49,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:49,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:49,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:49,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:49,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:49,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:49,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:49,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:49,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036489245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:49,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:49,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:49,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:49,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036489247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:49,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036489246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:49,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036489247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:49,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:49,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:49,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036489255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:49,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T17:13:49,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:49,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:49,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:49,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:49,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:49,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:49,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T17:13:49,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/412a1bba31264b349a9732ae7b8883d4 2024-11-19T17:13:49,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:49,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036489455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:49,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:49,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:49,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036489465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:49,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:49,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T17:13:49,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036489465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:49,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:49,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:49,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:49,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:49,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:49,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:49,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036489466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:49,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:49,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:49,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:49,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036489475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:49,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:49,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/6d29c6a1872d4fbf99fb5b889c0af1a7 is 50, key is test_row_0/B:col10/1732036428817/Put/seqid=0 2024-11-19T17:13:49,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741838_1014 (size=12001) 2024-11-19T17:13:49,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/6d29c6a1872d4fbf99fb5b889c0af1a7 2024-11-19T17:13:49,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/cc8da696192f4cbab47a81a1dc4b18ae is 50, key is test_row_0/C:col10/1732036428817/Put/seqid=0 2024-11-19T17:13:49,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T17:13:49,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:49,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:49,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:49,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:49,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:49,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:49,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741839_1015 (size=12001) 2024-11-19T17:13:49,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:49,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036489774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:49,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:49,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T17:13:49,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:49,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:49,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:49,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:49,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:49,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:49,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:49,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036489784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:49,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:49,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:49,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036489784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:49,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:49,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:49,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036489785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:49,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:49,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:49,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036489794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:49,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:49,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:49,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T17:13:49,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T17:13:49,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:49,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:49,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:49,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:49,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:49,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:50,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/cc8da696192f4cbab47a81a1dc4b18ae 2024-11-19T17:13:50,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/412a1bba31264b349a9732ae7b8883d4 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/412a1bba31264b349a9732ae7b8883d4 2024-11-19T17:13:50,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T17:13:50,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:50,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:50,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:50,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:50,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:50,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/412a1bba31264b349a9732ae7b8883d4, entries=150, sequenceid=13, filesize=11.7 K 2024-11-19T17:13:50,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/6d29c6a1872d4fbf99fb5b889c0af1a7 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/6d29c6a1872d4fbf99fb5b889c0af1a7 2024-11-19T17:13:50,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/6d29c6a1872d4fbf99fb5b889c0af1a7, entries=150, sequenceid=13, filesize=11.7 K 2024-11-19T17:13:50,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/cc8da696192f4cbab47a81a1dc4b18ae as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/cc8da696192f4cbab47a81a1dc4b18ae 2024-11-19T17:13:50,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/cc8da696192f4cbab47a81a1dc4b18ae, entries=150, sequenceid=13, filesize=11.7 K 2024-11-19T17:13:50,159 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for 689491498ff99efc87002cc59bc70ef3 in 1330ms, sequenceid=13, compaction requested=false 2024-11-19T17:13:50,160 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T17:13:50,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:50,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T17:13:50,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:50,257 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-19T17:13:50,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:13:50,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:13:50,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:13:50,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:13:50,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:13:50,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:13:50,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/0ada863ef623421fb57eeda223ef7e7e is 50, key is test_row_0/A:col10/1732036429065/Put/seqid=0 2024-11-19T17:13:50,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:50,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:50,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741840_1016 (size=12001) 2024-11-19T17:13:50,323 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/0ada863ef623421fb57eeda223ef7e7e 2024-11-19T17:13:50,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/4293bbf4b68f4fc895a6cfec00943ed7 is 50, key is test_row_0/B:col10/1732036429065/Put/seqid=0 2024-11-19T17:13:50,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:50,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036490351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:50,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:50,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036490351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:50,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:50,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:50,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:50,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036490354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:50,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:50,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:50,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036490356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:50,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:50,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036490359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:50,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:50,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741841_1017 (size=12001) 2024-11-19T17:13:50,396 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/4293bbf4b68f4fc895a6cfec00943ed7 2024-11-19T17:13:50,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/a89c4701d5334e999ae4acc0fe01c599 is 50, key is test_row_0/C:col10/1732036429065/Put/seqid=0 2024-11-19T17:13:50,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741842_1018 (size=12001) 2024-11-19T17:13:50,436 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/a89c4701d5334e999ae4acc0fe01c599 2024-11-19T17:13:50,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/0ada863ef623421fb57eeda223ef7e7e as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/0ada863ef623421fb57eeda223ef7e7e 2024-11-19T17:13:50,468 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/0ada863ef623421fb57eeda223ef7e7e, entries=150, sequenceid=48, filesize=11.7 K 2024-11-19T17:13:50,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/4293bbf4b68f4fc895a6cfec00943ed7 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/4293bbf4b68f4fc895a6cfec00943ed7 2024-11-19T17:13:50,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:50,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036490475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:50,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:50,490 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/4293bbf4b68f4fc895a6cfec00943ed7, entries=150, sequenceid=48, filesize=11.7 K 2024-11-19T17:13:50,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:50,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036490484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:50,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:50,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/a89c4701d5334e999ae4acc0fe01c599 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/a89c4701d5334e999ae4acc0fe01c599 2024-11-19T17:13:50,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:50,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036490485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:50,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:50,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:50,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036490485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:50,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:50,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:50,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036490485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:50,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:50,509 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/a89c4701d5334e999ae4acc0fe01c599, entries=150, sequenceid=48, filesize=11.7 K 2024-11-19T17:13:50,513 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 689491498ff99efc87002cc59bc70ef3 in 257ms, sequenceid=48, compaction requested=false 2024-11-19T17:13:50,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:50,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:50,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-19T17:13:50,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-19T17:13:50,523 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-19T17:13:50,523 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7490 sec 2024-11-19T17:13:50,530 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.7780 sec 2024-11-19T17:13:50,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T17:13:50,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:13:50,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:50,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:13:50,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:50,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:13:50,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:50,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:50,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/85712a4f27bf4cff916dbd01fbb2f3e4 is 50, key is test_row_0/A:col10/1732036430352/Put/seqid=0 2024-11-19T17:13:50,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036490750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:50,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:50,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741843_1019 (size=12001) 2024-11-19T17:13:50,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/85712a4f27bf4cff916dbd01fbb2f3e4 2024-11-19T17:13:50,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036490754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:50,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036490756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:50,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036490756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:50,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:50,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036490760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:50,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:50,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:50,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:50,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/eb0a4526815a4ab3813ad56038f36352 is 50, key is test_row_0/B:col10/1732036430352/Put/seqid=0 2024-11-19T17:13:50,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741844_1020 (size=12001) 2024-11-19T17:13:50,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/eb0a4526815a4ab3813ad56038f36352 2024-11-19T17:13:50,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/92d6f7532dc446d1b0706a077a2944c6 is 50, key is test_row_0/C:col10/1732036430352/Put/seqid=0 2024-11-19T17:13:50,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:50,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036490867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:50,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:50,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:50,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036490884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:50,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:50,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036490885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:50,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:50,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:50,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:50,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036490885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:50,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:50,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036490885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:50,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741845_1021 (size=12001) 2024-11-19T17:13:50,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/92d6f7532dc446d1b0706a077a2944c6 2024-11-19T17:13:50,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T17:13:50,914 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:13:50,915 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:13:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-19T17:13:50,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T17:13:50,925 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:13:50,927 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:13:50,927 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:13:50,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/85712a4f27bf4cff916dbd01fbb2f3e4 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/85712a4f27bf4cff916dbd01fbb2f3e4 2024-11-19T17:13:50,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/85712a4f27bf4cff916dbd01fbb2f3e4, entries=150, sequenceid=60, filesize=11.7 K 2024-11-19T17:13:50,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/eb0a4526815a4ab3813ad56038f36352 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/eb0a4526815a4ab3813ad56038f36352 2024-11-19T17:13:50,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/eb0a4526815a4ab3813ad56038f36352, entries=150, sequenceid=60, filesize=11.7 K 2024-11-19T17:13:50,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/92d6f7532dc446d1b0706a077a2944c6 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/92d6f7532dc446d1b0706a077a2944c6 2024-11-19T17:13:50,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/92d6f7532dc446d1b0706a077a2944c6, entries=150, sequenceid=60, filesize=11.7 K 2024-11-19T17:13:50,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 689491498ff99efc87002cc59bc70ef3 in 298ms, sequenceid=60, compaction requested=true 2024-11-19T17:13:50,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:50,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:13:50,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:13:50,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:13:50,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:13:50,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:13:50,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:13:50,996 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:13:50,996 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:13:51,000 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:13:51,002 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/B is initiating minor compaction (all files) 2024-11-19T17:13:51,002 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/B in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:51,003 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/6d29c6a1872d4fbf99fb5b889c0af1a7, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/4293bbf4b68f4fc895a6cfec00943ed7, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/eb0a4526815a4ab3813ad56038f36352] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=35.2 K 2024-11-19T17:13:51,004 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:13:51,005 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d29c6a1872d4fbf99fb5b889c0af1a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732036428803 2024-11-19T17:13:51,005 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/A is initiating minor compaction (all files) 2024-11-19T17:13:51,005 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/A in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:51,005 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 4293bbf4b68f4fc895a6cfec00943ed7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732036429065 2024-11-19T17:13:51,006 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/412a1bba31264b349a9732ae7b8883d4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/0ada863ef623421fb57eeda223ef7e7e, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/85712a4f27bf4cff916dbd01fbb2f3e4] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=35.2 K 2024-11-19T17:13:51,006 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting eb0a4526815a4ab3813ad56038f36352, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732036430350 2024-11-19T17:13:51,011 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 412a1bba31264b349a9732ae7b8883d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732036428803 2024-11-19T17:13:51,012 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ada863ef623421fb57eeda223ef7e7e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732036429065 2024-11-19T17:13:51,013 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85712a4f27bf4cff916dbd01fbb2f3e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732036430350 2024-11-19T17:13:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T17:13:51,060 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#B#compaction#9 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:13:51,061 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/81f05dcf80994b19940050dd4fd2a741 is 50, key is test_row_0/B:col10/1732036430352/Put/seqid=0 2024-11-19T17:13:51,070 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#A#compaction#10 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:13:51,071 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/2be3c46b72b643fbab1edc2e6dcfc906 is 50, key is test_row_0/A:col10/1732036430352/Put/seqid=0 2024-11-19T17:13:51,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T17:13:51,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:51,086 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T17:13:51,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:13:51,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:51,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:13:51,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:51,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:13:51,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:51,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741846_1022 (size=12104) 2024-11-19T17:13:51,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:51,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741847_1023 (size=12104) 2024-11-19T17:13:51,120 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/81f05dcf80994b19940050dd4fd2a741 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/81f05dcf80994b19940050dd4fd2a741 2024-11-19T17:13:51,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/7249596e0ace49678916e53cfb1967ae is 50, key is test_row_0/A:col10/1732036430749/Put/seqid=0 2024-11-19T17:13:51,135 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/2be3c46b72b643fbab1edc2e6dcfc906 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/2be3c46b72b643fbab1edc2e6dcfc906 2024-11-19T17:13:51,158 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/B of 689491498ff99efc87002cc59bc70ef3 into 81f05dcf80994b19940050dd4fd2a741(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:13:51,158 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:51,162 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/B, priority=13, startTime=1732036430995; duration=0sec 2024-11-19T17:13:51,163 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:13:51,164 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:B 2024-11-19T17:13:51,164 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/A of 689491498ff99efc87002cc59bc70ef3 into 2be3c46b72b643fbab1edc2e6dcfc906(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:13:51,164 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:51,164 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/A, priority=13, startTime=1732036430989; duration=0sec 2024-11-19T17:13:51,164 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:13:51,164 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:13:51,164 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:A 2024-11-19T17:13:51,169 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:13:51,170 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/C is initiating minor compaction (all files) 2024-11-19T17:13:51,170 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/C in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:51,171 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/cc8da696192f4cbab47a81a1dc4b18ae, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/a89c4701d5334e999ae4acc0fe01c599, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/92d6f7532dc446d1b0706a077a2944c6] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=35.2 K 2024-11-19T17:13:51,171 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting cc8da696192f4cbab47a81a1dc4b18ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732036428803 2024-11-19T17:13:51,172 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting a89c4701d5334e999ae4acc0fe01c599, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732036429065 2024-11-19T17:13:51,173 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 92d6f7532dc446d1b0706a077a2944c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732036430350 2024-11-19T17:13:51,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741848_1024 (size=12001) 2024-11-19T17:13:51,179 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/7249596e0ace49678916e53cfb1967ae 2024-11-19T17:13:51,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:51,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036491162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:51,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:51,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036491185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:51,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:51,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036491191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:51,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:51,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036491198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:51,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036491199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:51,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:51,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/8d7e31c4f3ec4f999a592e15f75900dc is 50, key is test_row_0/B:col10/1732036430749/Put/seqid=0 2024-11-19T17:13:51,226 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#C#compaction#13 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:13:51,227 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/5dc03f97a5cf45659bf93b6865e66d4b is 50, key is test_row_0/C:col10/1732036430352/Put/seqid=0 2024-11-19T17:13:51,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T17:13:51,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741849_1025 (size=12001) 2024-11-19T17:13:51,259 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/8d7e31c4f3ec4f999a592e15f75900dc 2024-11-19T17:13:51,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741850_1026 (size=12104) 2024-11-19T17:13:51,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/166069766cc34275b4f48a3e36533d30 is 50, key is test_row_0/C:col10/1732036430749/Put/seqid=0 2024-11-19T17:13:51,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036491304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:51,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:51,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741851_1027 (size=12001) 2024-11-19T17:13:51,316 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/166069766cc34275b4f48a3e36533d30 2024-11-19T17:13:51,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:51,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036491314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:51,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:51,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036491314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:51,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:51,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:51,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:51,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036491315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:51,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:51,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/7249596e0ace49678916e53cfb1967ae as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/7249596e0ace49678916e53cfb1967ae 2024-11-19T17:13:51,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036491324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:51,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:51,347 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/7249596e0ace49678916e53cfb1967ae, entries=150, sequenceid=80, filesize=11.7 K 2024-11-19T17:13:51,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/8d7e31c4f3ec4f999a592e15f75900dc as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/8d7e31c4f3ec4f999a592e15f75900dc 2024-11-19T17:13:51,363 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/8d7e31c4f3ec4f999a592e15f75900dc, entries=150, sequenceid=80, filesize=11.7 K 2024-11-19T17:13:51,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/166069766cc34275b4f48a3e36533d30 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/166069766cc34275b4f48a3e36533d30 2024-11-19T17:13:51,388 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/166069766cc34275b4f48a3e36533d30, entries=150, sequenceid=80, filesize=11.7 K 2024-11-19T17:13:51,390 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 689491498ff99efc87002cc59bc70ef3 in 303ms, sequenceid=80, compaction requested=false 2024-11-19T17:13:51,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:51,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:51,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-19T17:13:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-19T17:13:51,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-19T17:13:51,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 466 msec 2024-11-19T17:13:51,403 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 485 msec 2024-11-19T17:13:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:51,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T17:13:51,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:13:51,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:51,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:13:51,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:51,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:13:51,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:51,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/fcac6f6e06b94bd6ae0958db743a9cec is 50, key is test_row_0/A:col10/1732036431526/Put/seqid=0 2024-11-19T17:13:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T17:13:51,555 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:13:51,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741852_1028 (size=14341) 2024-11-19T17:13:51,555 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:13:51,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/fcac6f6e06b94bd6ae0958db743a9cec 2024-11-19T17:13:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-19T17:13:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T17:13:51,561 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:13:51,564 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:13:51,564 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:13:51,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/eee8fda4e1804de3a28dc947254b49d6 is 50, key is test_row_0/B:col10/1732036431526/Put/seqid=0 2024-11-19T17:13:51,578 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T17:13:51,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741853_1029 (size=12001) 2024-11-19T17:13:51,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/eee8fda4e1804de3a28dc947254b49d6 2024-11-19T17:13:51,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/f8fa352c0ca64395bbc955a8915ca0a2 is 50, key is test_row_0/C:col10/1732036431526/Put/seqid=0 2024-11-19T17:13:51,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741854_1030 (size=12001) 2024-11-19T17:13:51,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/f8fa352c0ca64395bbc955a8915ca0a2 2024-11-19T17:13:51,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/fcac6f6e06b94bd6ae0958db743a9cec as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/fcac6f6e06b94bd6ae0958db743a9cec 2024-11-19T17:13:51,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/fcac6f6e06b94bd6ae0958db743a9cec, entries=200, sequenceid=96, filesize=14.0 K 2024-11-19T17:13:51,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/eee8fda4e1804de3a28dc947254b49d6 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/eee8fda4e1804de3a28dc947254b49d6 2024-11-19T17:13:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T17:13:51,664 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-19T17:13:51,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:51,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036491653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:51,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/eee8fda4e1804de3a28dc947254b49d6, entries=150, sequenceid=96, filesize=11.7 K 2024-11-19T17:13:51,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036491653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:51,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:51,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/f8fa352c0ca64395bbc955a8915ca0a2 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/f8fa352c0ca64395bbc955a8915ca0a2 2024-11-19T17:13:51,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036491665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:51,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:51,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036491673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:51,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:51,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036491673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:51,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:51,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/f8fa352c0ca64395bbc955a8915ca0a2, entries=150, sequenceid=96, filesize=11.7 K 2024-11-19T17:13:51,700 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/5dc03f97a5cf45659bf93b6865e66d4b as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/5dc03f97a5cf45659bf93b6865e66d4b 2024-11-19T17:13:51,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 689491498ff99efc87002cc59bc70ef3 in 171ms, sequenceid=96, compaction requested=true 2024-11-19T17:13:51,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:51,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:A, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:13:51,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:13:51,701 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:13:51,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:B, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:13:51,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:13:51,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:13:51,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T17:13:51,703 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:13:51,703 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/A is initiating minor compaction (all files) 2024-11-19T17:13:51,703 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/A in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:51,704 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/2be3c46b72b643fbab1edc2e6dcfc906, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/7249596e0ace49678916e53cfb1967ae, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/fcac6f6e06b94bd6ae0958db743a9cec] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=37.5 K 2024-11-19T17:13:51,707 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2be3c46b72b643fbab1edc2e6dcfc906, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732036430350 2024-11-19T17:13:51,708 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7249596e0ace49678916e53cfb1967ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732036430744 2024-11-19T17:13:51,709 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcac6f6e06b94bd6ae0958db743a9cec, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732036431185 2024-11-19T17:13:51,717 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/C of 689491498ff99efc87002cc59bc70ef3 into 5dc03f97a5cf45659bf93b6865e66d4b(size=11.8 K), total size for store is 35.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:13:51,717 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:51,717 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/C, priority=13, startTime=1732036430995; duration=0sec 2024-11-19T17:13:51,717 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T17:13:51,717 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:C 2024-11-19T17:13:51,718 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:C 2024-11-19T17:13:51,718 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:13:51,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T17:13:51,721 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:13:51,721 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/B is initiating minor compaction (all files) 2024-11-19T17:13:51,721 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/B in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:51,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:51,721 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/81f05dcf80994b19940050dd4fd2a741, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/8d7e31c4f3ec4f999a592e15f75900dc, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/eee8fda4e1804de3a28dc947254b49d6] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=35.3 K 2024-11-19T17:13:51,721 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T17:13:51,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:13:51,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:51,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:13:51,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:51,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:13:51,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:51,723 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 81f05dcf80994b19940050dd4fd2a741, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732036430350 2024-11-19T17:13:51,727 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d7e31c4f3ec4f999a592e15f75900dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732036430744 2024-11-19T17:13:51,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/e7b7b24a474145f294896866a61ed477 is 50, key is test_row_0/A:col10/1732036431660/Put/seqid=0 2024-11-19T17:13:51,732 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting eee8fda4e1804de3a28dc947254b49d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732036431188 2024-11-19T17:13:51,745 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#A#compaction#19 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:13:51,746 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/f967a0c4215244cd93bf40c72d558b9c is 50, key is test_row_0/A:col10/1732036431526/Put/seqid=0 2024-11-19T17:13:51,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741855_1031 (size=12001) 2024-11-19T17:13:51,771 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#B#compaction#20 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:13:51,772 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/4dd0ecf1fadd436f9ef2080a93fa34af is 50, key is test_row_0/B:col10/1732036431526/Put/seqid=0 2024-11-19T17:13:51,773 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/e7b7b24a474145f294896866a61ed477 2024-11-19T17:13:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:51,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:51,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741856_1032 (size=12207) 2024-11-19T17:13:51,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/f4f7b01b13594de196666d254479b8c7 is 50, key is test_row_0/B:col10/1732036431660/Put/seqid=0 2024-11-19T17:13:51,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741857_1033 (size=12207) 2024-11-19T17:13:51,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036491811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:51,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036491807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:51,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:51,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:51,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:51,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036491818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:51,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:51,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:51,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:51,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036491824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:51,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036491823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:51,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:51,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:51,834 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/4dd0ecf1fadd436f9ef2080a93fa34af as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/4dd0ecf1fadd436f9ef2080a93fa34af 2024-11-19T17:13:51,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741858_1034 (size=12001) 2024-11-19T17:13:51,842 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/f4f7b01b13594de196666d254479b8c7 2024-11-19T17:13:51,858 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/B of 689491498ff99efc87002cc59bc70ef3 into 4dd0ecf1fadd436f9ef2080a93fa34af(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:13:51,858 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:51,858 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/B, priority=13, startTime=1732036431701; duration=0sec 2024-11-19T17:13:51,859 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:13:51,859 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:B 2024-11-19T17:13:51,859 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:13:51,864 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:13:51,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/4b371b28decc49909b316e97f58462ac is 50, key is test_row_0/C:col10/1732036431660/Put/seqid=0 2024-11-19T17:13:51,864 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/C is initiating minor compaction (all files) 2024-11-19T17:13:51,864 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/C in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:51,865 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/5dc03f97a5cf45659bf93b6865e66d4b, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/166069766cc34275b4f48a3e36533d30, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/f8fa352c0ca64395bbc955a8915ca0a2] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=35.3 K 2024-11-19T17:13:51,865 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dc03f97a5cf45659bf93b6865e66d4b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732036430350 2024-11-19T17:13:51,866 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 166069766cc34275b4f48a3e36533d30, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732036430744 2024-11-19T17:13:51,867 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting f8fa352c0ca64395bbc955a8915ca0a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732036431188 2024-11-19T17:13:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T17:13:51,883 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#C#compaction#23 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:13:51,884 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/e100b01ecbb04678b859edad0035a8de is 50, key is test_row_0/C:col10/1732036431526/Put/seqid=0 2024-11-19T17:13:51,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741859_1035 (size=12001) 2024-11-19T17:13:51,904 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/4b371b28decc49909b316e97f58462ac 2024-11-19T17:13:51,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741860_1036 (size=12207) 2024-11-19T17:13:51,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/e7b7b24a474145f294896866a61ed477 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/e7b7b24a474145f294896866a61ed477 2024-11-19T17:13:51,937 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/e7b7b24a474145f294896866a61ed477, entries=150, sequenceid=116, filesize=11.7 K 2024-11-19T17:13:51,938 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/e100b01ecbb04678b859edad0035a8de as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/e100b01ecbb04678b859edad0035a8de 2024-11-19T17:13:51,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/f4f7b01b13594de196666d254479b8c7 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/f4f7b01b13594de196666d254479b8c7 2024-11-19T17:13:51,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:51,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:51,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036491935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:51,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036491935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:51,950 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/C of 689491498ff99efc87002cc59bc70ef3 into e100b01ecbb04678b859edad0035a8de(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:13:51,951 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:51,951 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/C, priority=13, startTime=1732036431701; duration=0sec 2024-11-19T17:13:51,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:51,951 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:13:51,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036491935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:51,951 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:C 2024-11-19T17:13:51,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:51,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:51,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:51,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036491935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:51,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:51,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:51,954 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/f4f7b01b13594de196666d254479b8c7, entries=150, sequenceid=116, filesize=11.7 K 2024-11-19T17:13:51,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:51,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036491935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:51,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:51,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:51,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/4b371b28decc49909b316e97f58462ac as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/4b371b28decc49909b316e97f58462ac 2024-11-19T17:13:51,967 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/4b371b28decc49909b316e97f58462ac, entries=150, sequenceid=116, filesize=11.7 K 2024-11-19T17:13:51,970 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 689491498ff99efc87002cc59bc70ef3 in 248ms, sequenceid=116, compaction requested=false 2024-11-19T17:13:51,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:51,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:51,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-19T17:13:51,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-19T17:13:51,978 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-19T17:13:51,978 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 410 msec 2024-11-19T17:13:51,982 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 425 msec 2024-11-19T17:13:52,133 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-19T17:13:52,134 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-19T17:13:52,136 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T17:13:52,136 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-19T17:13:52,136 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-19T17:13:52,136 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-19T17:13:52,138 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T17:13:52,138 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-19T17:13:52,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:52,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T17:13:52,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:13:52,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:52,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:13:52,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:52,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:13:52,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T17:13:52,184 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:13:52,185 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:13:52,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-19T17:13:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T17:13:52,189 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:13:52,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/124027852d094f65b31f1a7f1366236a is 50, key is test_row_0/A:col10/1732036432165/Put/seqid=0 2024-11-19T17:13:52,190 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:13:52,190 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:13:52,201 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/f967a0c4215244cd93bf40c72d558b9c as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/f967a0c4215244cd93bf40c72d558b9c 2024-11-19T17:13:52,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741861_1037 (size=12101) 2024-11-19T17:13:52,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/124027852d094f65b31f1a7f1366236a 2024-11-19T17:13:52,218 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/A of 689491498ff99efc87002cc59bc70ef3 into f967a0c4215244cd93bf40c72d558b9c(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:13:52,218 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:52,218 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/A, priority=13, startTime=1732036431701; duration=0sec 2024-11-19T17:13:52,218 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:13:52,219 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:A 2024-11-19T17:13:52,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/31cd7e26846f4b2480e5697e9ae629ef is 50, key is test_row_0/B:col10/1732036432165/Put/seqid=0 2024-11-19T17:13:52,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741862_1038 (size=12101) 2024-11-19T17:13:52,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036492243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036492248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:52,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:52,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:52,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036492248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:52,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:52,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036492252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:52,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:52,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036492252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:52,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T17:13:52,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T17:13:52,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:52,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:52,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:52,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:52,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:52,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:52,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036492365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036492364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:52,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:52,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:52,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036492365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036492365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:52,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:52,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:52,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:52,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:52,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036492373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:52,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:52,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T17:13:52,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:52,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:52,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:52,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:52,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T17:13:52,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036492574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:52,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036492575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:52,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:52,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036492575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:52,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:52,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036492575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:52,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:52,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036492584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:52,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:52,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T17:13:52,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:52,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:52,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:52,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:52,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:52,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/31cd7e26846f4b2480e5697e9ae629ef 2024-11-19T17:13:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:52,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/3da0a5910587412f8c2ac89ea54e53b6 is 50, key is test_row_0/C:col10/1732036432165/Put/seqid=0 2024-11-19T17:13:52,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741863_1039 (size=12101) 2024-11-19T17:13:52,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/3da0a5910587412f8c2ac89ea54e53b6 2024-11-19T17:13:52,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/124027852d094f65b31f1a7f1366236a as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/124027852d094f65b31f1a7f1366236a 2024-11-19T17:13:52,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/124027852d094f65b31f1a7f1366236a, entries=150, sequenceid=134, filesize=11.8 K 2024-11-19T17:13:52,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/31cd7e26846f4b2480e5697e9ae629ef as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/31cd7e26846f4b2480e5697e9ae629ef 2024-11-19T17:13:52,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/31cd7e26846f4b2480e5697e9ae629ef, entries=150, sequenceid=134, filesize=11.8 K 2024-11-19T17:13:52,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/3da0a5910587412f8c2ac89ea54e53b6 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/3da0a5910587412f8c2ac89ea54e53b6 2024-11-19T17:13:52,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/3da0a5910587412f8c2ac89ea54e53b6, entries=150, sequenceid=134, filesize=11.8 K 2024-11-19T17:13:52,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 689491498ff99efc87002cc59bc70ef3 in 604ms, sequenceid=134, compaction requested=true 2024-11-19T17:13:52,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:52,773 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:13:52,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:13:52,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:13:52,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:13:52,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:13:52,775 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:13:52,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:13:52,775 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:13:52,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:13:52,775 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/A is initiating minor compaction (all files) 2024-11-19T17:13:52,775 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/A in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:52,775 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/f967a0c4215244cd93bf40c72d558b9c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/e7b7b24a474145f294896866a61ed477, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/124027852d094f65b31f1a7f1366236a] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=35.5 K 2024-11-19T17:13:52,776 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting f967a0c4215244cd93bf40c72d558b9c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732036431188 2024-11-19T17:13:52,777 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7b7b24a474145f294896866a61ed477, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732036431660 2024-11-19T17:13:52,778 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:13:52,778 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/B is initiating minor compaction (all files) 2024-11-19T17:13:52,778 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 124027852d094f65b31f1a7f1366236a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732036431813 2024-11-19T17:13:52,778 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/B in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:52,778 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/4dd0ecf1fadd436f9ef2080a93fa34af, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/f4f7b01b13594de196666d254479b8c7, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/31cd7e26846f4b2480e5697e9ae629ef] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=35.5 K 2024-11-19T17:13:52,780 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dd0ecf1fadd436f9ef2080a93fa34af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732036431188 2024-11-19T17:13:52,780 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting f4f7b01b13594de196666d254479b8c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732036431660 2024-11-19T17:13:52,781 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 31cd7e26846f4b2480e5697e9ae629ef, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732036431813 2024-11-19T17:13:52,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T17:13:52,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T17:13:52,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:52,820 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T17:13:52,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:13:52,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:52,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:13:52,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:52,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:13:52,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:52,822 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#A#compaction#27 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:13:52,823 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/288ea15fb8d143979acb82930193f480 is 50, key is test_row_0/A:col10/1732036432165/Put/seqid=0 2024-11-19T17:13:52,828 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#B#compaction#28 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:13:52,829 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/b5cbc35d021a4d82a51b7148d4f6feb4 is 50, key is test_row_0/B:col10/1732036432165/Put/seqid=0 2024-11-19T17:13:52,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741864_1040 (size=12409) 2024-11-19T17:13:52,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/a85b36766c0945c390d9ee817ab51d4e is 50, key is test_row_0/A:col10/1732036432247/Put/seqid=0 2024-11-19T17:13:52,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741865_1041 (size=12409) 2024-11-19T17:13:52,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741866_1042 (size=12151) 2024-11-19T17:13:52,853 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/a85b36766c0945c390d9ee817ab51d4e 2024-11-19T17:13:52,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/5006982c214c4fdcb9a8573d516b3455 is 50, key is test_row_0/B:col10/1732036432247/Put/seqid=0 2024-11-19T17:13:52,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:52,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:52,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741867_1043 (size=12151) 2024-11-19T17:13:52,899 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/5006982c214c4fdcb9a8573d516b3455 2024-11-19T17:13:52,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/6a84aca6083342f88410477c5a9996a5 is 50, key is test_row_0/C:col10/1732036432247/Put/seqid=0 2024-11-19T17:13:52,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036492915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:52,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:52,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:52,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036492918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:52,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036492921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:52,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036492915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:52,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:52,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036492930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:52,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:52,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:52,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:52,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741868_1044 (size=12151) 2024-11-19T17:13:52,950 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/6a84aca6083342f88410477c5a9996a5 2024-11-19T17:13:52,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/a85b36766c0945c390d9ee817ab51d4e as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/a85b36766c0945c390d9ee817ab51d4e 2024-11-19T17:13:52,973 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/a85b36766c0945c390d9ee817ab51d4e, entries=150, sequenceid=151, filesize=11.9 K 2024-11-19T17:13:52,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/5006982c214c4fdcb9a8573d516b3455 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/5006982c214c4fdcb9a8573d516b3455 2024-11-19T17:13:52,985 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/5006982c214c4fdcb9a8573d516b3455, entries=150, sequenceid=151, filesize=11.9 K 2024-11-19T17:13:52,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/6a84aca6083342f88410477c5a9996a5 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/6a84aca6083342f88410477c5a9996a5 2024-11-19T17:13:52,997 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/6a84aca6083342f88410477c5a9996a5, entries=150, sequenceid=151, filesize=11.9 K 2024-11-19T17:13:52,999 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 689491498ff99efc87002cc59bc70ef3 in 178ms, sequenceid=151, compaction requested=true 2024-11-19T17:13:52,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:52,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:52,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-19T17:13:53,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-19T17:13:53,006 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-19T17:13:53,006 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 812 msec 2024-11-19T17:13:53,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 823 msec 2024-11-19T17:13:53,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:53,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T17:13:53,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:13:53,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:53,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:13:53,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:53,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:13:53,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:53,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/72806d71bc8e48c994d86445874b540f is 50, key is test_row_0/A:col10/1732036433036/Put/seqid=0 2024-11-19T17:13:53,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741869_1045 (size=14541) 2024-11-19T17:13:53,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/72806d71bc8e48c994d86445874b540f 2024-11-19T17:13:53,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:53,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:53,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036493070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:53,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036493074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:53,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:53,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:53,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036493079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:53,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:53,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:53,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036493077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:53,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:53,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:53,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:53,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036493082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:53,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:53,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/8f500f1c53304da08352ef8421324b2a is 50, key is test_row_0/B:col10/1732036433036/Put/seqid=0 2024-11-19T17:13:53,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741870_1046 (size=12151) 2024-11-19T17:13:53,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/8f500f1c53304da08352ef8421324b2a 2024-11-19T17:13:53,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/c39cbaa5cf4643819d474a2766b24087 is 50, key is test_row_0/C:col10/1732036433036/Put/seqid=0 2024-11-19T17:13:53,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741871_1047 (size=12151) 2024-11-19T17:13:53,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/c39cbaa5cf4643819d474a2766b24087 2024-11-19T17:13:53,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/72806d71bc8e48c994d86445874b540f as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/72806d71bc8e48c994d86445874b540f 2024-11-19T17:13:53,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/72806d71bc8e48c994d86445874b540f, entries=200, sequenceid=168, filesize=14.2 K 2024-11-19T17:13:53,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/8f500f1c53304da08352ef8421324b2a as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/8f500f1c53304da08352ef8421324b2a 2024-11-19T17:13:53,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/8f500f1c53304da08352ef8421324b2a, entries=150, sequenceid=168, filesize=11.9 K 2024-11-19T17:13:53,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/c39cbaa5cf4643819d474a2766b24087 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/c39cbaa5cf4643819d474a2766b24087 2024-11-19T17:13:53,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/c39cbaa5cf4643819d474a2766b24087, entries=150, sequenceid=168, filesize=11.9 K 2024-11-19T17:13:53,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 689491498ff99efc87002cc59bc70ef3 in 155ms, sequenceid=168, compaction requested=true 2024-11-19T17:13:53,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:53,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:A, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:13:53,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T17:13:53,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:B, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:13:53,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-19T17:13:53,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:13:53,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-19T17:13:53,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:53,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T17:13:53,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:13:53,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:53,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:13:53,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:53,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:13:53,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:53,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/2dabb56049354be38dd1b7845157abfd is 50, key is test_row_0/A:col10/1732036433201/Put/seqid=0 2024-11-19T17:13:53,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:53,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036493222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:53,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:53,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:53,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036493224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:53,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:53,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:53,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036493229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:53,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:53,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741872_1048 (size=16931) 2024-11-19T17:13:53,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/2dabb56049354be38dd1b7845157abfd 2024-11-19T17:13:53,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:53,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036493232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:53,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:53,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:53,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036493234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:53,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:53,258 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/288ea15fb8d143979acb82930193f480 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/288ea15fb8d143979acb82930193f480 2024-11-19T17:13:53,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/a27909cc86144dd8a67d2aa3ff9a9ff9 is 50, key is test_row_0/B:col10/1732036433201/Put/seqid=0 2024-11-19T17:13:53,263 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/b5cbc35d021a4d82a51b7148d4f6feb4 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/b5cbc35d021a4d82a51b7148d4f6feb4 2024-11-19T17:13:53,271 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/A of 689491498ff99efc87002cc59bc70ef3 into 288ea15fb8d143979acb82930193f480(size=12.1 K), total size for store is 38.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:13:53,271 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:53,271 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/A, priority=13, startTime=1732036432773; duration=0sec 2024-11-19T17:13:53,272 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-19T17:13:53,272 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:A 2024-11-19T17:13:53,272 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:A 2024-11-19T17:13:53,272 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T17:13:53,274 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/B of 689491498ff99efc87002cc59bc70ef3 into b5cbc35d021a4d82a51b7148d4f6feb4(size=12.1 K), total size for store is 35.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:13:53,274 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:53,274 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/B, priority=13, startTime=1732036432774; duration=0sec 2024-11-19T17:13:53,274 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-19T17:13:53,274 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:B 2024-11-19T17:13:53,274 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:B 2024-11-19T17:13:53,274 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:13:53,275 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60611 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T17:13:53,275 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/C is initiating minor compaction (all files) 2024-11-19T17:13:53,275 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/C in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:53,276 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/e100b01ecbb04678b859edad0035a8de, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/4b371b28decc49909b316e97f58462ac, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/3da0a5910587412f8c2ac89ea54e53b6, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/6a84aca6083342f88410477c5a9996a5, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/c39cbaa5cf4643819d474a2766b24087] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=59.2 K 2024-11-19T17:13:53,277 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39101 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:13:53,277 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/A is initiating minor compaction (all files) 2024-11-19T17:13:53,277 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting e100b01ecbb04678b859edad0035a8de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732036431188 2024-11-19T17:13:53,277 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/A in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:53,277 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/288ea15fb8d143979acb82930193f480, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/a85b36766c0945c390d9ee817ab51d4e, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/72806d71bc8e48c994d86445874b540f] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=38.2 K 2024-11-19T17:13:53,277 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b371b28decc49909b316e97f58462ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732036431660 2024-11-19T17:13:53,278 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 288ea15fb8d143979acb82930193f480, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732036431813 2024-11-19T17:13:53,278 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3da0a5910587412f8c2ac89ea54e53b6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732036431813 2024-11-19T17:13:53,279 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting a85b36766c0945c390d9ee817ab51d4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732036432242 2024-11-19T17:13:53,279 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a84aca6083342f88410477c5a9996a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732036432242 2024-11-19T17:13:53,279 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 72806d71bc8e48c994d86445874b540f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732036432902 2024-11-19T17:13:53,280 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting c39cbaa5cf4643819d474a2766b24087, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732036432902 2024-11-19T17:13:53,307 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#A#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:13:53,308 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/c42694a1dd5d48b4abc5c1a2fe6deeb2 is 50, key is test_row_0/A:col10/1732036433036/Put/seqid=0 2024-11-19T17:13:53,316 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#C#compaction#38 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:13:53,317 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/721c9cbc2ce24532b4d82b771892c98b is 50, key is test_row_0/C:col10/1732036433036/Put/seqid=0 2024-11-19T17:13:53,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T17:13:53,325 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:13:53,326 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:13:53,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-19T17:13:53,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T17:13:53,329 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:13:53,332 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:13:53,332 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:13:53,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:53,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036493334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:53,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:53,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741873_1049 (size=12151) 2024-11-19T17:13:53,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741875_1051 (size=12561) 2024-11-19T17:13:53,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:53,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036493344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:53,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:53,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:53,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036493344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:53,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:53,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741874_1050 (size=12527) 2024-11-19T17:13:53,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:53,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036493354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:53,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:53,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:53,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036493355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:53,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:53,373 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/c42694a1dd5d48b4abc5c1a2fe6deeb2 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/c42694a1dd5d48b4abc5c1a2fe6deeb2 2024-11-19T17:13:53,384 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/A of 689491498ff99efc87002cc59bc70ef3 into c42694a1dd5d48b4abc5c1a2fe6deeb2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:13:53,384 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:53,384 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/A, priority=13, startTime=1732036433194; duration=0sec 2024-11-19T17:13:53,384 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T17:13:53,384 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:A 2024-11-19T17:13:53,384 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:13:53,386 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:13:53,386 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/B is initiating minor compaction (all files) 2024-11-19T17:13:53,386 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/B in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:53,386 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/b5cbc35d021a4d82a51b7148d4f6feb4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/5006982c214c4fdcb9a8573d516b3455, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/8f500f1c53304da08352ef8421324b2a] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=35.9 K 2024-11-19T17:13:53,387 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting b5cbc35d021a4d82a51b7148d4f6feb4, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732036431813 2024-11-19T17:13:53,387 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 5006982c214c4fdcb9a8573d516b3455, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732036432242 2024-11-19T17:13:53,388 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f500f1c53304da08352ef8421324b2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732036432902 2024-11-19T17:13:53,400 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#B#compaction#39 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:13:53,401 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/223e780d2b5d4d0797a279e74e87eb01 is 50, key is test_row_0/B:col10/1732036433036/Put/seqid=0 2024-11-19T17:13:53,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741876_1052 (size=12561) 2024-11-19T17:13:53,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T17:13:53,435 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/223e780d2b5d4d0797a279e74e87eb01 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/223e780d2b5d4d0797a279e74e87eb01 2024-11-19T17:13:53,447 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/B of 689491498ff99efc87002cc59bc70ef3 into 223e780d2b5d4d0797a279e74e87eb01(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:13:53,447 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:53,447 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/B, priority=13, startTime=1732036433194; duration=0sec 2024-11-19T17:13:53,447 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:13:53,448 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:B 2024-11-19T17:13:53,448 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 5 compacting, 0 eligible, 16 blocking 2024-11-19T17:13:53,448 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-19T17:13:53,448 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-19T17:13:53,449 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. because compaction request was cancelled 2024-11-19T17:13:53,450 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:C 2024-11-19T17:13:53,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T17:13:53,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:53,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:53,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:53,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:53,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:53,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:53,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:53,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036493544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:53,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:53,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:53,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:53,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036493564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:53,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036493564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:53,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:53,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:53,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:53,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036493574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:53,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:53,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:53,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036493574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:53,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T17:13:53,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T17:13:53,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:53,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:53,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:53,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:53,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:53,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:53,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/a27909cc86144dd8a67d2aa3ff9a9ff9 2024-11-19T17:13:53,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/55da356648e94aabb9e98d5eb68d5fab is 50, key is test_row_0/C:col10/1732036433201/Put/seqid=0 2024-11-19T17:13:53,764 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/721c9cbc2ce24532b4d82b771892c98b as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/721c9cbc2ce24532b4d82b771892c98b 2024-11-19T17:13:53,773 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/C of 689491498ff99efc87002cc59bc70ef3 into 721c9cbc2ce24532b4d82b771892c98b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:13:53,773 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:53,773 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/C, priority=11, startTime=1732036433194; duration=0sec 2024-11-19T17:13:53,773 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:13:53,773 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:C 2024-11-19T17:13:53,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741877_1053 (size=12151) 2024-11-19T17:13:53,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/55da356648e94aabb9e98d5eb68d5fab 2024-11-19T17:13:53,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/2dabb56049354be38dd1b7845157abfd as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/2dabb56049354be38dd1b7845157abfd 2024-11-19T17:13:53,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/2dabb56049354be38dd1b7845157abfd, entries=250, sequenceid=188, filesize=16.5 K 2024-11-19T17:13:53,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/a27909cc86144dd8a67d2aa3ff9a9ff9 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/a27909cc86144dd8a67d2aa3ff9a9ff9 2024-11-19T17:13:53,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/a27909cc86144dd8a67d2aa3ff9a9ff9, entries=150, sequenceid=188, filesize=11.9 K 2024-11-19T17:13:53,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/55da356648e94aabb9e98d5eb68d5fab as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/55da356648e94aabb9e98d5eb68d5fab 2024-11-19T17:13:53,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T17:13:53,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/55da356648e94aabb9e98d5eb68d5fab, entries=150, sequenceid=188, filesize=11.9 K 2024-11-19T17:13:53,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:53,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:53,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:53,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:53,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:53,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:53,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 689491498ff99efc87002cc59bc70ef3 in 619ms, sequenceid=188, compaction requested=false 2024-11-19T17:13:53,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:53,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:53,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T17:13:53,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:13:53,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:53,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:13:53,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:53,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:13:53,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:53,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/a28a4e42c44d4f998239fc14d7501016 is 50, key is test_row_0/A:col10/1732036433856/Put/seqid=0 2024-11-19T17:13:53,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741878_1054 (size=14541) 2024-11-19T17:13:53,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/a28a4e42c44d4f998239fc14d7501016 2024-11-19T17:13:53,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:53,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036493899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:53,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:53,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:53,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036493902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:53,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:53,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:53,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036493904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:53,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:53,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:53,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036493907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:53,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:53,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:53,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036493907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:53,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/71a38e6db75f4db68b263df2e9eb85ab is 50, key is test_row_0/B:col10/1732036433856/Put/seqid=0 2024-11-19T17:13:53,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741879_1055 (size=12151) 2024-11-19T17:13:53,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/71a38e6db75f4db68b263df2e9eb85ab 2024-11-19T17:13:53,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T17:13:53,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/9ac9b1fb30bc40a2bb7bdc5b62b5a8ff is 50, key is test_row_0/C:col10/1732036433856/Put/seqid=0 2024-11-19T17:13:53,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T17:13:53,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:53,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:53,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:53,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:53,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:53,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:53,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741880_1056 (size=12151) 2024-11-19T17:13:54,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:54,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036494014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:54,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:54,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036494014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:54,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:54,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:54,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036494015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:54,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:54,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:54,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036494024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:54,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:54,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:54,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036494024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:54,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:54,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T17:13:54,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:54,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:54,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:54,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:54,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:54,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036494224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:54,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:54,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036494224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:54,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:54,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036494224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:54,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:54,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:54,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036494234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:54,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:54,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036494234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:54,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:54,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:54,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T17:13:54,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:54,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:54,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:54,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:54,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:54,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/9ac9b1fb30bc40a2bb7bdc5b62b5a8ff 2024-11-19T17:13:54,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/a28a4e42c44d4f998239fc14d7501016 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/a28a4e42c44d4f998239fc14d7501016 2024-11-19T17:13:54,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/a28a4e42c44d4f998239fc14d7501016, entries=200, sequenceid=206, filesize=14.2 K 2024-11-19T17:13:54,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/71a38e6db75f4db68b263df2e9eb85ab as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/71a38e6db75f4db68b263df2e9eb85ab 2024-11-19T17:13:54,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/71a38e6db75f4db68b263df2e9eb85ab, entries=150, sequenceid=206, filesize=11.9 K 2024-11-19T17:13:54,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/9ac9b1fb30bc40a2bb7bdc5b62b5a8ff as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/9ac9b1fb30bc40a2bb7bdc5b62b5a8ff 2024-11-19T17:13:54,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/9ac9b1fb30bc40a2bb7bdc5b62b5a8ff, entries=150, sequenceid=206, filesize=11.9 K 2024-11-19T17:13:54,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 689491498ff99efc87002cc59bc70ef3 in 560ms, sequenceid=206, compaction requested=true 2024-11-19T17:13:54,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:54,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:13:54,419 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:13:54,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:13:54,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:13:54,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:13:54,419 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:13:54,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:13:54,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:13:54,420 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44033 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:13:54,420 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:13:54,420 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/B is initiating minor compaction (all files) 2024-11-19T17:13:54,420 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/A is initiating minor compaction (all files) 2024-11-19T17:13:54,420 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/B in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:54,420 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/A in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:54,421 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/223e780d2b5d4d0797a279e74e87eb01, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/a27909cc86144dd8a67d2aa3ff9a9ff9, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/71a38e6db75f4db68b263df2e9eb85ab] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=36.0 K 2024-11-19T17:13:54,421 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/c42694a1dd5d48b4abc5c1a2fe6deeb2, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/2dabb56049354be38dd1b7845157abfd, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/a28a4e42c44d4f998239fc14d7501016] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=43.0 K 2024-11-19T17:13:54,421 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 223e780d2b5d4d0797a279e74e87eb01, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732036432902 2024-11-19T17:13:54,421 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting c42694a1dd5d48b4abc5c1a2fe6deeb2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732036432902 2024-11-19T17:13:54,422 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2dabb56049354be38dd1b7845157abfd, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732036433073 2024-11-19T17:13:54,422 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting a27909cc86144dd8a67d2aa3ff9a9ff9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732036433073 2024-11-19T17:13:54,423 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting a28a4e42c44d4f998239fc14d7501016, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732036433217 2024-11-19T17:13:54,423 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 71a38e6db75f4db68b263df2e9eb85ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732036433217 2024-11-19T17:13:54,438 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#A#compaction#44 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:13:54,438 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#B#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:13:54,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T17:13:54,441 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/06db09cc1e7144a7b7649ad2d2997d2e is 50, key is test_row_0/A:col10/1732036433856/Put/seqid=0 2024-11-19T17:13:54,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:54,441 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T17:13:54,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:13:54,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:54,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:13:54,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:54,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:13:54,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:54,443 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/56a4aae1a0cd43c482e2f719e3738040 is 50, key is test_row_0/B:col10/1732036433856/Put/seqid=0 2024-11-19T17:13:54,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741881_1057 (size=12663) 2024-11-19T17:13:54,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/ffd3d2cacbce4bacaa833b8f875e8cab is 50, key is test_row_0/A:col10/1732036433904/Put/seqid=0 2024-11-19T17:13:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T17:13:54,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741882_1058 (size=12663) 2024-11-19T17:13:54,480 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/56a4aae1a0cd43c482e2f719e3738040 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/56a4aae1a0cd43c482e2f719e3738040 2024-11-19T17:13:54,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741883_1059 (size=12151) 2024-11-19T17:13:54,494 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/ffd3d2cacbce4bacaa833b8f875e8cab 2024-11-19T17:13:54,495 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/B of 689491498ff99efc87002cc59bc70ef3 into 56a4aae1a0cd43c482e2f719e3738040(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:13:54,495 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:54,495 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/B, priority=13, startTime=1732036434419; duration=0sec 2024-11-19T17:13:54,495 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:13:54,495 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:B 2024-11-19T17:13:54,495 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:13:54,501 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:13:54,501 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/C is initiating minor compaction (all files) 2024-11-19T17:13:54,501 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/C in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:54,502 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/721c9cbc2ce24532b4d82b771892c98b, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/55da356648e94aabb9e98d5eb68d5fab, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/9ac9b1fb30bc40a2bb7bdc5b62b5a8ff] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=36.0 K 2024-11-19T17:13:54,503 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 721c9cbc2ce24532b4d82b771892c98b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732036432902 2024-11-19T17:13:54,503 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 55da356648e94aabb9e98d5eb68d5fab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732036433073 2024-11-19T17:13:54,505 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ac9b1fb30bc40a2bb7bdc5b62b5a8ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732036433217 2024-11-19T17:13:54,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/81049fa87db540359ca84976179d05d6 is 50, key is test_row_0/B:col10/1732036433904/Put/seqid=0 2024-11-19T17:13:54,527 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#C#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:13:54,528 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/75068864d62f4d779e6a8c5163ae0e8c is 50, key is test_row_0/C:col10/1732036433856/Put/seqid=0 2024-11-19T17:13:54,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:54,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741884_1060 (size=12151) 2024-11-19T17:13:54,550 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/81049fa87db540359ca84976179d05d6 2024-11-19T17:13:54,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741885_1061 (size=12629) 2024-11-19T17:13:54,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:54,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036494558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:54,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:54,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:54,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036494560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:54,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:54,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:54,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036494562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:54,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:54,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:54,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036494562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:54,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:54,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/9f2217e83a6b4ba1a876ea683ac9df49 is 50, key is test_row_0/C:col10/1732036433904/Put/seqid=0 2024-11-19T17:13:54,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036494565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:54,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:54,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741886_1062 (size=12151) 2024-11-19T17:13:54,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036494674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:54,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:54,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036494674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:54,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:54,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036494674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:54,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:54,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036494674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:54,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:54,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036494684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:54,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:54,857 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/06db09cc1e7144a7b7649ad2d2997d2e as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/06db09cc1e7144a7b7649ad2d2997d2e 2024-11-19T17:13:54,870 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/A of 689491498ff99efc87002cc59bc70ef3 into 06db09cc1e7144a7b7649ad2d2997d2e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:13:54,870 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:54,870 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/A, priority=13, startTime=1732036434419; duration=0sec 2024-11-19T17:13:54,871 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:13:54,871 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:A 2024-11-19T17:13:54,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036494884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:54,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036494884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:54,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:54,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036494884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036494885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:54,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:54,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036494894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:54,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:54,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:54,961 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/75068864d62f4d779e6a8c5163ae0e8c as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/75068864d62f4d779e6a8c5163ae0e8c 2024-11-19T17:13:54,970 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/C of 689491498ff99efc87002cc59bc70ef3 into 75068864d62f4d779e6a8c5163ae0e8c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:13:54,970 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:54,970 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/C, priority=13, startTime=1732036434419; duration=0sec 2024-11-19T17:13:54,972 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:13:54,972 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:C 2024-11-19T17:13:54,991 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/9f2217e83a6b4ba1a876ea683ac9df49 2024-11-19T17:13:55,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/ffd3d2cacbce4bacaa833b8f875e8cab as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/ffd3d2cacbce4bacaa833b8f875e8cab 2024-11-19T17:13:55,009 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/ffd3d2cacbce4bacaa833b8f875e8cab, entries=150, sequenceid=225, filesize=11.9 K 2024-11-19T17:13:55,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/81049fa87db540359ca84976179d05d6 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/81049fa87db540359ca84976179d05d6 2024-11-19T17:13:55,017 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/81049fa87db540359ca84976179d05d6, entries=150, sequenceid=225, filesize=11.9 K 2024-11-19T17:13:55,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/9f2217e83a6b4ba1a876ea683ac9df49 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/9f2217e83a6b4ba1a876ea683ac9df49 2024-11-19T17:13:55,025 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/9f2217e83a6b4ba1a876ea683ac9df49, entries=150, sequenceid=225, filesize=11.9 K 2024-11-19T17:13:55,028 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 689491498ff99efc87002cc59bc70ef3 in 587ms, sequenceid=225, compaction requested=false 2024-11-19T17:13:55,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:55,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:55,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-19T17:13:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-19T17:13:55,033 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-19T17:13:55,034 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6980 sec 2024-11-19T17:13:55,037 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.7080 sec 2024-11-19T17:13:55,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:55,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T17:13:55,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:13:55,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:55,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:13:55,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:55,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:13:55,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:55,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/bcf4efd778ce4c858f601e910e479989 is 50, key is test_row_0/A:col10/1732036434562/Put/seqid=0 2024-11-19T17:13:55,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:55,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036495222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:55,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:55,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036495223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:55,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:55,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:55,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:55,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036495224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:55,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:55,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:55,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036495226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:55,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:55,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:55,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741887_1063 (size=12151) 2024-11-19T17:13:55,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036495230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:55,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:55,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/bcf4efd778ce4c858f601e910e479989 2024-11-19T17:13:55,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/874535960c08428c96d2fac89d7eb6f2 is 50, key is test_row_0/B:col10/1732036434562/Put/seqid=0 2024-11-19T17:13:55,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741888_1064 (size=12151) 2024-11-19T17:13:55,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/874535960c08428c96d2fac89d7eb6f2 2024-11-19T17:13:55,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/6c7ac8a6cb614729bef8c602d6eb5275 is 50, key is test_row_0/C:col10/1732036434562/Put/seqid=0 2024-11-19T17:13:55,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741889_1065 (size=12151) 2024-11-19T17:13:55,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/6c7ac8a6cb614729bef8c602d6eb5275 2024-11-19T17:13:55,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:55,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036495334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:55,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:55,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036495334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:55,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:55,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:55,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036495334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:55,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:55,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:55,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:55,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036495343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:55,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:55,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:55,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036495344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:55,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:55,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/bcf4efd778ce4c858f601e910e479989 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/bcf4efd778ce4c858f601e910e479989 2024-11-19T17:13:55,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/bcf4efd778ce4c858f601e910e479989, entries=150, sequenceid=243, filesize=11.9 K 2024-11-19T17:13:55,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/874535960c08428c96d2fac89d7eb6f2 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/874535960c08428c96d2fac89d7eb6f2 2024-11-19T17:13:55,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/874535960c08428c96d2fac89d7eb6f2, entries=150, sequenceid=243, filesize=11.9 K 2024-11-19T17:13:55,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/6c7ac8a6cb614729bef8c602d6eb5275 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/6c7ac8a6cb614729bef8c602d6eb5275 2024-11-19T17:13:55,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/6c7ac8a6cb614729bef8c602d6eb5275, entries=150, sequenceid=243, filesize=11.9 K 2024-11-19T17:13:55,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 689491498ff99efc87002cc59bc70ef3 in 186ms, sequenceid=243, compaction requested=true 2024-11-19T17:13:55,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:55,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:13:55,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:13:55,384 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:13:55,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:13:55,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:13:55,384 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:13:55,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:13:55,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:13:55,385 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:13:55,385 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:13:55,386 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/B is initiating minor compaction (all files) 2024-11-19T17:13:55,386 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/A is initiating minor compaction (all files) 2024-11-19T17:13:55,386 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/B in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:55,386 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/A in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:55,386 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/56a4aae1a0cd43c482e2f719e3738040, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/81049fa87db540359ca84976179d05d6, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/874535960c08428c96d2fac89d7eb6f2] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=36.1 K 2024-11-19T17:13:55,386 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/06db09cc1e7144a7b7649ad2d2997d2e, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/ffd3d2cacbce4bacaa833b8f875e8cab, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/bcf4efd778ce4c858f601e910e479989] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=36.1 K 2024-11-19T17:13:55,386 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 56a4aae1a0cd43c482e2f719e3738040, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732036433217 2024-11-19T17:13:55,386 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06db09cc1e7144a7b7649ad2d2997d2e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732036433217 2024-11-19T17:13:55,387 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 81049fa87db540359ca84976179d05d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732036433901 2024-11-19T17:13:55,387 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffd3d2cacbce4bacaa833b8f875e8cab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732036433901 2024-11-19T17:13:55,387 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 874535960c08428c96d2fac89d7eb6f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732036434562 2024-11-19T17:13:55,387 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcf4efd778ce4c858f601e910e479989, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732036434562 2024-11-19T17:13:55,407 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#A#compaction#53 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:13:55,408 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/497374115a8b4a6d872ca3a2b2dcc1a8 is 50, key is test_row_0/A:col10/1732036434562/Put/seqid=0 2024-11-19T17:13:55,421 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#B#compaction#54 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:13:55,422 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/b21d7f1fb8644180806a3f5a38742ea5 is 50, key is test_row_0/B:col10/1732036434562/Put/seqid=0 2024-11-19T17:13:55,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741890_1066 (size=12765) 2024-11-19T17:13:55,464 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/497374115a8b4a6d872ca3a2b2dcc1a8 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/497374115a8b4a6d872ca3a2b2dcc1a8 2024-11-19T17:13:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T17:13:55,476 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:13:55,477 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:13:55,478 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/A of 689491498ff99efc87002cc59bc70ef3 into 497374115a8b4a6d872ca3a2b2dcc1a8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:13:55,478 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:55,478 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/A, priority=13, startTime=1732036435384; duration=0sec 2024-11-19T17:13:55,478 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:13:55,478 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:A 2024-11-19T17:13:55,478 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:13:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-19T17:13:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T17:13:55,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741891_1067 (size=12765) 2024-11-19T17:13:55,481 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:13:55,481 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/C is initiating minor compaction (all files) 2024-11-19T17:13:55,481 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/C in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:55,481 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/75068864d62f4d779e6a8c5163ae0e8c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/9f2217e83a6b4ba1a876ea683ac9df49, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/6c7ac8a6cb614729bef8c602d6eb5275] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=36.1 K 2024-11-19T17:13:55,482 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75068864d62f4d779e6a8c5163ae0e8c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732036433217 2024-11-19T17:13:55,482 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:13:55,482 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f2217e83a6b4ba1a876ea683ac9df49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732036433901 2024-11-19T17:13:55,484 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c7ac8a6cb614729bef8c602d6eb5275, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732036434562 2024-11-19T17:13:55,485 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:13:55,486 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:13:55,494 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/b21d7f1fb8644180806a3f5a38742ea5 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/b21d7f1fb8644180806a3f5a38742ea5 2024-11-19T17:13:55,503 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/B of 689491498ff99efc87002cc59bc70ef3 into b21d7f1fb8644180806a3f5a38742ea5(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:13:55,504 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:55,504 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/B, priority=13, startTime=1732036435384; duration=0sec 2024-11-19T17:13:55,504 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:13:55,504 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:B 2024-11-19T17:13:55,510 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#C#compaction#55 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:13:55,510 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/489e641fd1e8449c94379a69f7c81fd8 is 50, key is test_row_0/C:col10/1732036434562/Put/seqid=0 2024-11-19T17:13:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:55,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T17:13:55,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:13:55,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:55,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:13:55,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:55,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:13:55,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:55,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741892_1068 (size=12731) 2024-11-19T17:13:55,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/d745487350e547b6a78ed62d436008a4 is 50, key is test_row_0/A:col10/1732036435224/Put/seqid=0 2024-11-19T17:13:55,571 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/489e641fd1e8449c94379a69f7c81fd8 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/489e641fd1e8449c94379a69f7c81fd8 2024-11-19T17:13:55,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741893_1069 (size=14691) 2024-11-19T17:13:55,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036495573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:55,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/d745487350e547b6a78ed62d436008a4 2024-11-19T17:13:55,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036495575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:55,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:55,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036495577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:55,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:55,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036495579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:55,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:55,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T17:13:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036495581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:55,587 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/C of 689491498ff99efc87002cc59bc70ef3 into 489e641fd1e8449c94379a69f7c81fd8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:13:55,587 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:55,587 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/C, priority=13, startTime=1732036435384; duration=0sec 2024-11-19T17:13:55,587 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:13:55,587 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:C 2024-11-19T17:13:55,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/8715444661c6438e8f3f849cabe024ac is 50, key is test_row_0/B:col10/1732036435224/Put/seqid=0 2024-11-19T17:13:55,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741894_1070 (size=12301) 2024-11-19T17:13:55,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/8715444661c6438e8f3f849cabe024ac 2024-11-19T17:13:55,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T17:13:55,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:55,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:55,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:55,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:55,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:55,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/edcb6f33a4a3465599e144a1c7fad27a is 50, key is test_row_0/C:col10/1732036435224/Put/seqid=0 2024-11-19T17:13:55,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741895_1071 (size=12301) 2024-11-19T17:13:55,666 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/edcb6f33a4a3465599e144a1c7fad27a 2024-11-19T17:13:55,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/d745487350e547b6a78ed62d436008a4 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/d745487350e547b6a78ed62d436008a4 2024-11-19T17:13:55,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/d745487350e547b6a78ed62d436008a4, entries=200, sequenceid=263, filesize=14.3 K 2024-11-19T17:13:55,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036495686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:55,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/8715444661c6438e8f3f849cabe024ac as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/8715444661c6438e8f3f849cabe024ac 2024-11-19T17:13:55,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:55,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036495686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:55,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:55,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/8715444661c6438e8f3f849cabe024ac, entries=150, sequenceid=263, filesize=12.0 K 2024-11-19T17:13:55,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/edcb6f33a4a3465599e144a1c7fad27a as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/edcb6f33a4a3465599e144a1c7fad27a 2024-11-19T17:13:55,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036495694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:55,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:55,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036495694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:55,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:55,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036495694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:55,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:55,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/edcb6f33a4a3465599e144a1c7fad27a, entries=150, sequenceid=263, filesize=12.0 K 2024-11-19T17:13:55,715 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 689491498ff99efc87002cc59bc70ef3 in 167ms, sequenceid=263, compaction requested=false 2024-11-19T17:13:55,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T17:13:55,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T17:13:55,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:55,798 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T17:13:55,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:13:55,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:55,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:13:55,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:55,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:13:55,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:55,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/6e691ee35a464b21a38c67f73b892968 is 50, key is test_row_0/A:col10/1732036435578/Put/seqid=0 2024-11-19T17:13:55,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741896_1072 (size=12301) 2024-11-19T17:13:55,827 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/6e691ee35a464b21a38c67f73b892968 2024-11-19T17:13:55,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/34b33fdb706e40e395473f116af943c9 is 50, key is test_row_0/B:col10/1732036435578/Put/seqid=0 2024-11-19T17:13:55,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741897_1073 (size=12301) 2024-11-19T17:13:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:55,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:55,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036495927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:55,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:55,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036495929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:55,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:55,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036495929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:55,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036495932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:55,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036495932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:56,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036496044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:56,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:56,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036496044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:56,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:56,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036496044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:56,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:56,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:56,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:56,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036496045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:56,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:56,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:56,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:56,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:56,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036496045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:56,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:56,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:56,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T17:13:56,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036496258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:56,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036496259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:56,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036496259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:56,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:56,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:56,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:56,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:56,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:56,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:56,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:56,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036496259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:56,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:56,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:56,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:56,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:56,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036496260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:56,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:56,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:56,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:56,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:56,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:56,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:56,280 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/34b33fdb706e40e395473f116af943c9 2024-11-19T17:13:56,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/aa7d720c06e44a8d87af784b9ff7064f is 50, key is test_row_0/C:col10/1732036435578/Put/seqid=0 2024-11-19T17:13:56,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741898_1074 (size=12301) 2024-11-19T17:13:56,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:56,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036496574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:56,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:56,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036496574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:56,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:56,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036496574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:56,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:56,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036496575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:56,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:56,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036496575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:56,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:56,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:56,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:56,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:56,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:56,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:56,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:56,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:56,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:56,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T17:13:56,725 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/aa7d720c06e44a8d87af784b9ff7064f 2024-11-19T17:13:56,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/6e691ee35a464b21a38c67f73b892968 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/6e691ee35a464b21a38c67f73b892968 2024-11-19T17:13:56,742 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/6e691ee35a464b21a38c67f73b892968, entries=150, sequenceid=279, filesize=12.0 K 2024-11-19T17:13:56,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/34b33fdb706e40e395473f116af943c9 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/34b33fdb706e40e395473f116af943c9 2024-11-19T17:13:56,749 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/34b33fdb706e40e395473f116af943c9, entries=150, sequenceid=279, filesize=12.0 K 2024-11-19T17:13:56,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/aa7d720c06e44a8d87af784b9ff7064f as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/aa7d720c06e44a8d87af784b9ff7064f 2024-11-19T17:13:56,755 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/aa7d720c06e44a8d87af784b9ff7064f, entries=150, sequenceid=279, filesize=12.0 K 2024-11-19T17:13:56,757 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 689491498ff99efc87002cc59bc70ef3 in 960ms, sequenceid=279, compaction requested=true 2024-11-19T17:13:56,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:56,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:56,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-19T17:13:56,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-19T17:13:56,763 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-19T17:13:56,763 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2740 sec 2024-11-19T17:13:56,765 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.2850 sec 2024-11-19T17:13:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:57,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T17:13:57,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:13:57,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:57,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:13:57,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:57,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:13:57,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:57,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/5d2254c632d74e83941979969445f7a8 is 50, key is test_row_0/A:col10/1732036435927/Put/seqid=0 2024-11-19T17:13:57,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741899_1075 (size=14741) 2024-11-19T17:13:57,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/5d2254c632d74e83941979969445f7a8 2024-11-19T17:13:57,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036497103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:57,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:57,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/54fb7313e9074b78aa72add90ca0dc8d is 50, key is test_row_0/B:col10/1732036435927/Put/seqid=0 2024-11-19T17:13:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036497104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:57,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036497106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036497109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:57,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:57,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036497110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:57,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:57,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741900_1076 (size=12301) 2024-11-19T17:13:57,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/54fb7313e9074b78aa72add90ca0dc8d 2024-11-19T17:13:57,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/2d216efd1aa4404e87ee0ea4d56d6923 is 50, key is test_row_0/C:col10/1732036435927/Put/seqid=0 2024-11-19T17:13:57,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741901_1077 (size=12301) 2024-11-19T17:13:57,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036497214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:57,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036497214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:57,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036497224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:57,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:57,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036497225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:57,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:57,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036497225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:57,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:57,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036497434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:57,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:57,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:57,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036497434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:57,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:57,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036497434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:57,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:57,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036497435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:57,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:57,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036497435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:57,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:57,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:57,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/2d216efd1aa4404e87ee0ea4d56d6923 2024-11-19T17:13:57,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/5d2254c632d74e83941979969445f7a8 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/5d2254c632d74e83941979969445f7a8 2024-11-19T17:13:57,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/5d2254c632d74e83941979969445f7a8, entries=200, sequenceid=298, filesize=14.4 K 2024-11-19T17:13:57,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/54fb7313e9074b78aa72add90ca0dc8d as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/54fb7313e9074b78aa72add90ca0dc8d 2024-11-19T17:13:57,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/54fb7313e9074b78aa72add90ca0dc8d, entries=150, sequenceid=298, filesize=12.0 K 2024-11-19T17:13:57,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/2d216efd1aa4404e87ee0ea4d56d6923 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/2d216efd1aa4404e87ee0ea4d56d6923 2024-11-19T17:13:57,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/2d216efd1aa4404e87ee0ea4d56d6923, entries=150, sequenceid=298, filesize=12.0 K 2024-11-19T17:13:57,612 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 689491498ff99efc87002cc59bc70ef3 in 526ms, sequenceid=298, compaction requested=true 2024-11-19T17:13:57,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:57,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:13:57,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:13:57,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:13:57,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:13:57,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:13:57,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:13:57,613 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:13:57,613 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:13:57,615 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:13:57,615 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/B is initiating minor compaction (all files) 2024-11-19T17:13:57,616 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/B in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:57,616 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/b21d7f1fb8644180806a3f5a38742ea5, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/8715444661c6438e8f3f849cabe024ac, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/34b33fdb706e40e395473f116af943c9, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/54fb7313e9074b78aa72add90ca0dc8d] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=48.5 K 2024-11-19T17:13:57,616 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54498 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:13:57,616 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/A is initiating minor compaction (all files) 2024-11-19T17:13:57,616 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/A in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:57,616 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/497374115a8b4a6d872ca3a2b2dcc1a8, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/d745487350e547b6a78ed62d436008a4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/6e691ee35a464b21a38c67f73b892968, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/5d2254c632d74e83941979969445f7a8] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=53.2 K 2024-11-19T17:13:57,618 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting b21d7f1fb8644180806a3f5a38742ea5, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732036434562 2024-11-19T17:13:57,618 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 497374115a8b4a6d872ca3a2b2dcc1a8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732036434562 2024-11-19T17:13:57,618 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting d745487350e547b6a78ed62d436008a4, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732036435219 2024-11-19T17:13:57,618 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 8715444661c6438e8f3f849cabe024ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732036435224 2024-11-19T17:13:57,619 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e691ee35a464b21a38c67f73b892968, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732036435570 2024-11-19T17:13:57,619 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 34b33fdb706e40e395473f116af943c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732036435570 2024-11-19T17:13:57,619 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d2254c632d74e83941979969445f7a8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732036435920 2024-11-19T17:13:57,620 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 54fb7313e9074b78aa72add90ca0dc8d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732036435920 2024-11-19T17:13:57,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T17:13:57,624 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:13:57,624 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:13:57,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-19T17:13:57,628 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:13:57,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T17:13:57,629 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:13:57,629 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:13:57,661 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#B#compaction#65 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:13:57,662 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/4935923d8b4647f4b9136be177b6f60c is 50, key is test_row_0/B:col10/1732036435927/Put/seqid=0 2024-11-19T17:13:57,668 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#A#compaction#66 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:13:57,669 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/e0c634da53794865b25c6ce2cffec44a is 50, key is test_row_0/A:col10/1732036435927/Put/seqid=0 2024-11-19T17:13:57,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741902_1078 (size=13051) 2024-11-19T17:13:57,708 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/4935923d8b4647f4b9136be177b6f60c as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/4935923d8b4647f4b9136be177b6f60c 2024-11-19T17:13:57,717 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/B of 689491498ff99efc87002cc59bc70ef3 into 4935923d8b4647f4b9136be177b6f60c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:13:57,717 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:57,717 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/B, priority=12, startTime=1732036437613; duration=0sec 2024-11-19T17:13:57,717 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:13:57,718 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:B 2024-11-19T17:13:57,718 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:13:57,720 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:13:57,720 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/C is initiating minor compaction (all files) 2024-11-19T17:13:57,720 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/C in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:57,721 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/489e641fd1e8449c94379a69f7c81fd8, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/edcb6f33a4a3465599e144a1c7fad27a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/aa7d720c06e44a8d87af784b9ff7064f, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/2d216efd1aa4404e87ee0ea4d56d6923] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=48.5 K 2024-11-19T17:13:57,722 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 489e641fd1e8449c94379a69f7c81fd8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732036434562 2024-11-19T17:13:57,723 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting edcb6f33a4a3465599e144a1c7fad27a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732036435224 2024-11-19T17:13:57,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741903_1079 (size=13051) 2024-11-19T17:13:57,724 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting aa7d720c06e44a8d87af784b9ff7064f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732036435570 2024-11-19T17:13:57,726 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d216efd1aa4404e87ee0ea4d56d6923, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732036435920 2024-11-19T17:13:57,732 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/e0c634da53794865b25c6ce2cffec44a as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/e0c634da53794865b25c6ce2cffec44a 2024-11-19T17:13:57,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T17:13:57,741 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/A of 689491498ff99efc87002cc59bc70ef3 into e0c634da53794865b25c6ce2cffec44a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:13:57,741 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:57,741 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/A, priority=12, startTime=1732036437613; duration=0sec 2024-11-19T17:13:57,741 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:13:57,742 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:A 2024-11-19T17:13:57,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:57,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T17:13:57,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:13:57,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:57,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:13:57,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:57,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:13:57,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:57,761 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#C#compaction#67 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:13:57,762 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/c2349cf18af241ab8559cbbfd4192a63 is 50, key is test_row_0/C:col10/1732036435927/Put/seqid=0 2024-11-19T17:13:57,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/bea616e0004b4ef3adc4cf36ff6dd8a8 is 50, key is test_row_0/A:col10/1732036437745/Put/seqid=0 2024-11-19T17:13:57,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T17:13:57,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:57,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:57,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:57,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:57,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:57,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:57,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:57,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036497783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:57,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:57,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:57,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036497785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:57,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:57,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:57,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036497787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:57,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:57,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:57,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036497788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:57,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:57,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036497789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:57,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:57,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:57,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741904_1080 (size=13017) 2024-11-19T17:13:57,809 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/c2349cf18af241ab8559cbbfd4192a63 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/c2349cf18af241ab8559cbbfd4192a63 2024-11-19T17:13:57,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741905_1081 (size=12297) 2024-11-19T17:13:57,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/bea616e0004b4ef3adc4cf36ff6dd8a8 2024-11-19T17:13:57,825 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/C of 689491498ff99efc87002cc59bc70ef3 into c2349cf18af241ab8559cbbfd4192a63(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:13:57,825 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:57,825 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/C, priority=12, startTime=1732036437613; duration=0sec 2024-11-19T17:13:57,826 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:13:57,826 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:C 2024-11-19T17:13:57,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/ec8874209090488dbe1e8371e7bda713 is 50, key is test_row_0/B:col10/1732036437745/Put/seqid=0 2024-11-19T17:13:57,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741906_1082 (size=9857) 2024-11-19T17:13:57,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:57,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036497904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:57,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:57,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036497904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:57,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:57,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:57,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:57,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036497904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:57,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:57,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:57,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036497904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:57,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:57,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036497904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:57,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:57,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T17:13:57,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:57,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:57,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:57,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:57,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:57,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:57,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T17:13:58,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T17:13:58,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:58,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:58,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:58,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:58,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:58,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:58,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:58,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036498114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:58,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:58,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:58,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:58,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036498114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:58,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036498114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:58,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036498115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:58,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:58,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036498124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:58,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:58,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T17:13:58,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:58,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:58,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:58,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:58,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:58,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:58,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T17:13:58,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/ec8874209090488dbe1e8371e7bda713 2024-11-19T17:13:58,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/2697b17a65034698b21227949bc17a7c is 50, key is test_row_0/C:col10/1732036437745/Put/seqid=0 2024-11-19T17:13:58,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741907_1083 (size=9857) 2024-11-19T17:13:58,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/2697b17a65034698b21227949bc17a7c 2024-11-19T17:13:58,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/bea616e0004b4ef3adc4cf36ff6dd8a8 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/bea616e0004b4ef3adc4cf36ff6dd8a8 2024-11-19T17:13:58,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/bea616e0004b4ef3adc4cf36ff6dd8a8, entries=150, sequenceid=314, filesize=12.0 K 2024-11-19T17:13:58,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/ec8874209090488dbe1e8371e7bda713 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/ec8874209090488dbe1e8371e7bda713 2024-11-19T17:13:58,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/ec8874209090488dbe1e8371e7bda713, entries=100, sequenceid=314, filesize=9.6 K 2024-11-19T17:13:58,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/2697b17a65034698b21227949bc17a7c as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/2697b17a65034698b21227949bc17a7c 2024-11-19T17:13:58,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/2697b17a65034698b21227949bc17a7c, entries=100, sequenceid=314, filesize=9.6 K 2024-11-19T17:13:58,371 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 689491498ff99efc87002cc59bc70ef3 in 623ms, sequenceid=314, compaction requested=false 2024-11-19T17:13:58,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:58,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T17:13:58,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:58,399 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T17:13:58,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:13:58,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:58,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:13:58,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:58,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:13:58,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:58,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/2f7baaa8744c493f97be3d75aa54e151 is 50, key is test_row_0/A:col10/1732036437778/Put/seqid=0 2024-11-19T17:13:58,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741908_1084 (size=12301) 2024-11-19T17:13:58,424 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/2f7baaa8744c493f97be3d75aa54e151 2024-11-19T17:13:58,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:58,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:58,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:58,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036498443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:58,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:58,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036498445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:58,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:58,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:58,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036498446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:58,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:58,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:58,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036498448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:58,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:58,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:58,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036498450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:58,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:58,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/6bb0b9acc6a843deb76a30799dfb8f45 is 50, key is test_row_0/B:col10/1732036437778/Put/seqid=0 2024-11-19T17:13:58,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741909_1085 (size=12301) 2024-11-19T17:13:58,477 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/6bb0b9acc6a843deb76a30799dfb8f45 2024-11-19T17:13:58,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/c01ab535b5e649608ed82fa436f5b38c is 50, key is test_row_0/C:col10/1732036437778/Put/seqid=0 2024-11-19T17:13:58,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741910_1086 (size=12301) 2024-11-19T17:13:58,519 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/c01ab535b5e649608ed82fa436f5b38c 2024-11-19T17:13:58,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/2f7baaa8744c493f97be3d75aa54e151 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/2f7baaa8744c493f97be3d75aa54e151 2024-11-19T17:13:58,537 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/2f7baaa8744c493f97be3d75aa54e151, entries=150, sequenceid=334, filesize=12.0 K 2024-11-19T17:13:58,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/6bb0b9acc6a843deb76a30799dfb8f45 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/6bb0b9acc6a843deb76a30799dfb8f45 2024-11-19T17:13:58,546 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/6bb0b9acc6a843deb76a30799dfb8f45, entries=150, sequenceid=334, filesize=12.0 K 2024-11-19T17:13:58,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/c01ab535b5e649608ed82fa436f5b38c as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/c01ab535b5e649608ed82fa436f5b38c 2024-11-19T17:13:58,557 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/c01ab535b5e649608ed82fa436f5b38c, entries=150, sequenceid=334, filesize=12.0 K 2024-11-19T17:13:58,558 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 689491498ff99efc87002cc59bc70ef3 in 159ms, sequenceid=334, compaction requested=true 2024-11-19T17:13:58,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:58,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:58,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-19T17:13:58,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-19T17:13:58,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-19T17:13:58,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 931 msec 2024-11-19T17:13:58,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T17:13:58,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:13:58,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:58,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:13:58,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:58,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:13:58,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:58,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:58,569 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 943 msec 2024-11-19T17:13:58,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/e461b5c52c9049e1bdc3686b3f129469 is 50, key is test_row_0/A:col10/1732036438427/Put/seqid=0 2024-11-19T17:13:58,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741911_1087 (size=12301) 2024-11-19T17:13:58,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:58,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036498600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:58,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:58,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:58,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036498601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:58,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:58,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:58,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036498605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:58,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:58,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:58,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036498618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:58,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:58,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:58,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036498634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:58,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:58,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036498724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:58,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:58,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036498725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:58,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:58,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036498736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:58,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:58,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:58,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036498745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:58,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:58,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:58,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036498745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:58,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:58,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T17:13:58,765 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:13:58,765 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:13:58,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-19T17:13:58,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T17:13:58,774 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:13:58,775 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:13:58,775 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:13:58,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T17:13:58,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T17:13:58,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:58,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:58,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:58,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:58,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:58,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:58,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:58,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036498934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:58,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:58,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:58,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:58,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036498944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:58,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036498944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:58,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:58,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:58,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:58,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036498964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:58,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:58,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:58,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036498964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:58,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:58,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:59,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/e461b5c52c9049e1bdc3686b3f129469 2024-11-19T17:13:59,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/80c22b9b22ad408cafd19ce39f5080be is 50, key is test_row_0/B:col10/1732036438427/Put/seqid=0 2024-11-19T17:13:59,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741912_1088 (size=12301) 2024-11-19T17:13:59,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/80c22b9b22ad408cafd19ce39f5080be 2024-11-19T17:13:59,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/7935e682368a45e39fab7c52c1ba5a6c is 50, key is test_row_0/C:col10/1732036438427/Put/seqid=0 2024-11-19T17:13:59,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741913_1089 (size=12301) 2024-11-19T17:13:59,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T17:13:59,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:59,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:59,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:59,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:59,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:59,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:59,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T17:13:59,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T17:13:59,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:59,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:59,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:59,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:59,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:59,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:59,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:59,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036499243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:59,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:59,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:59,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:59,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036499254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:59,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:59,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:59,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:59,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:59,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036499254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:59,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:59,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:59,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:59,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:59,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:59,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036499273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:59,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036499273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:59,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:59,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:59,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:59,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:59,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:59,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:59,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T17:13:59,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:59,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:59,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:59,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:59,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:59,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:13:59,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T17:13:59,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/7935e682368a45e39fab7c52c1ba5a6c 2024-11-19T17:13:59,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/e461b5c52c9049e1bdc3686b3f129469 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/e461b5c52c9049e1bdc3686b3f129469 2024-11-19T17:13:59,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/e461b5c52c9049e1bdc3686b3f129469, entries=150, sequenceid=349, filesize=12.0 K 2024-11-19T17:13:59,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/80c22b9b22ad408cafd19ce39f5080be as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/80c22b9b22ad408cafd19ce39f5080be 2024-11-19T17:13:59,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/80c22b9b22ad408cafd19ce39f5080be, entries=150, sequenceid=349, filesize=12.0 K 2024-11-19T17:13:59,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/7935e682368a45e39fab7c52c1ba5a6c as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/7935e682368a45e39fab7c52c1ba5a6c 2024-11-19T17:13:59,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/7935e682368a45e39fab7c52c1ba5a6c, entries=150, sequenceid=349, filesize=12.0 K 2024-11-19T17:13:59,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=107.34 KB/109920 for 689491498ff99efc87002cc59bc70ef3 in 947ms, sequenceid=349, compaction requested=true 2024-11-19T17:13:59,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:59,516 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:13:59,517 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49950 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:13:59,517 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/A is initiating minor compaction (all files) 2024-11-19T17:13:59,517 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/A in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:59,518 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/e0c634da53794865b25c6ce2cffec44a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/bea616e0004b4ef3adc4cf36ff6dd8a8, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/2f7baaa8744c493f97be3d75aa54e151, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/e461b5c52c9049e1bdc3686b3f129469] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=48.8 K 2024-11-19T17:13:59,518 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0c634da53794865b25c6ce2cffec44a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732036435920 2024-11-19T17:13:59,519 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting bea616e0004b4ef3adc4cf36ff6dd8a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732036437104 2024-11-19T17:13:59,519 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f7baaa8744c493f97be3d75aa54e151, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732036437778 2024-11-19T17:13:59,519 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting e461b5c52c9049e1bdc3686b3f129469, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732036438427 2024-11-19T17:13:59,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:13:59,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:13:59,522 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:13:59,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:13:59,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:13:59,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:13:59,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:13:59,524 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:13:59,524 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/B is initiating minor compaction (all files) 2024-11-19T17:13:59,525 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/B in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:59,525 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/4935923d8b4647f4b9136be177b6f60c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/ec8874209090488dbe1e8371e7bda713, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/6bb0b9acc6a843deb76a30799dfb8f45, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/80c22b9b22ad408cafd19ce39f5080be] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=46.4 K 2024-11-19T17:13:59,525 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 4935923d8b4647f4b9136be177b6f60c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732036435920 2024-11-19T17:13:59,526 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting ec8874209090488dbe1e8371e7bda713, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732036437104 2024-11-19T17:13:59,527 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bb0b9acc6a843deb76a30799dfb8f45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732036437778 2024-11-19T17:13:59,528 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 80c22b9b22ad408cafd19ce39f5080be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732036438427 2024-11-19T17:13:59,540 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#A#compaction#77 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:13:59,541 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/afa60b4eae5b4849b42b83d32b47f42b is 50, key is test_row_0/A:col10/1732036438427/Put/seqid=0 2024-11-19T17:13:59,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T17:13:59,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:59,546 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T17:13:59,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:13:59,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:59,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:13:59,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:59,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:13:59,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:13:59,556 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#B#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:13:59,557 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/105f79f2123f4f99952b16acf9e92fab is 50, key is test_row_0/B:col10/1732036438427/Put/seqid=0 2024-11-19T17:13:59,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/db845d7b4f354ceab7b3fbac65437a91 is 50, key is test_row_0/A:col10/1732036438597/Put/seqid=0 2024-11-19T17:13:59,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741914_1090 (size=13187) 2024-11-19T17:13:59,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741915_1091 (size=13187) 2024-11-19T17:13:59,580 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/105f79f2123f4f99952b16acf9e92fab as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/105f79f2123f4f99952b16acf9e92fab 2024-11-19T17:13:59,590 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/B of 689491498ff99efc87002cc59bc70ef3 into 105f79f2123f4f99952b16acf9e92fab(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:13:59,590 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:59,590 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/B, priority=12, startTime=1732036439522; duration=0sec 2024-11-19T17:13:59,591 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:13:59,591 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:B 2024-11-19T17:13:59,591 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:13:59,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741916_1092 (size=12301) 2024-11-19T17:13:59,596 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/db845d7b4f354ceab7b3fbac65437a91 2024-11-19T17:13:59,598 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:13:59,598 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/C is initiating minor compaction (all files) 2024-11-19T17:13:59,598 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/C in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:13:59,599 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/c2349cf18af241ab8559cbbfd4192a63, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/2697b17a65034698b21227949bc17a7c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/c01ab535b5e649608ed82fa436f5b38c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/7935e682368a45e39fab7c52c1ba5a6c] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=46.4 K 2024-11-19T17:13:59,599 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting c2349cf18af241ab8559cbbfd4192a63, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732036435920 2024-11-19T17:13:59,600 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 2697b17a65034698b21227949bc17a7c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732036437104 2024-11-19T17:13:59,601 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting c01ab535b5e649608ed82fa436f5b38c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732036437778 2024-11-19T17:13:59,602 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 7935e682368a45e39fab7c52c1ba5a6c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732036438427 2024-11-19T17:13:59,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/778389b0eee74594b8f0c0f2fd52ce3f is 50, key is test_row_0/B:col10/1732036438597/Put/seqid=0 2024-11-19T17:13:59,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741917_1093 (size=12301) 2024-11-19T17:13:59,640 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#C#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:13:59,641 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/34d9471bddd84730ac1f342ef5a219ed is 50, key is test_row_0/C:col10/1732036438427/Put/seqid=0 2024-11-19T17:13:59,654 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/778389b0eee74594b8f0c0f2fd52ce3f 2024-11-19T17:13:59,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741918_1094 (size=13153) 2024-11-19T17:13:59,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/03e2261148df447db2e791124dce802c is 50, key is test_row_0/C:col10/1732036438597/Put/seqid=0 2024-11-19T17:13:59,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741919_1095 (size=12301) 2024-11-19T17:13:59,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:13:59,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:13:59,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:59,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:59,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036499784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:59,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036499780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:59,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:59,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:59,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:59,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:59,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:59,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036499787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:59,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:59,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036499789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:59,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:59,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:59,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036499794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:59,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:59,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:59,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:59,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:59,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036499894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:59,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:59,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:59,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:59,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036499903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:59,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:13:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036499903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:13:59,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:59,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:13:59,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:13:59,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T17:13:59,972 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/afa60b4eae5b4849b42b83d32b47f42b as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/afa60b4eae5b4849b42b83d32b47f42b 2024-11-19T17:13:59,978 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/A of 689491498ff99efc87002cc59bc70ef3 into afa60b4eae5b4849b42b83d32b47f42b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:13:59,978 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:13:59,978 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/A, priority=12, startTime=1732036439515; duration=0sec 2024-11-19T17:13:59,978 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:13:59,978 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:A 2024-11-19T17:14:00,070 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/34d9471bddd84730ac1f342ef5a219ed as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/34d9471bddd84730ac1f342ef5a219ed 2024-11-19T17:14:00,077 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/C of 689491498ff99efc87002cc59bc70ef3 into 34d9471bddd84730ac1f342ef5a219ed(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:00,077 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:00,077 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/C, priority=12, startTime=1732036439523; duration=0sec 2024-11-19T17:14:00,077 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:00,077 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:C 2024-11-19T17:14:00,086 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/03e2261148df447db2e791124dce802c 2024-11-19T17:14:00,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/db845d7b4f354ceab7b3fbac65437a91 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/db845d7b4f354ceab7b3fbac65437a91 2024-11-19T17:14:00,097 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/db845d7b4f354ceab7b3fbac65437a91, entries=150, sequenceid=368, filesize=12.0 K 2024-11-19T17:14:00,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/778389b0eee74594b8f0c0f2fd52ce3f as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/778389b0eee74594b8f0c0f2fd52ce3f 2024-11-19T17:14:00,103 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/778389b0eee74594b8f0c0f2fd52ce3f, entries=150, sequenceid=368, filesize=12.0 K 2024-11-19T17:14:00,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/03e2261148df447db2e791124dce802c as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/03e2261148df447db2e791124dce802c 2024-11-19T17:14:00,110 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/03e2261148df447db2e791124dce802c, entries=150, sequenceid=368, filesize=12.0 K 2024-11-19T17:14:00,111 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 689491498ff99efc87002cc59bc70ef3 in 565ms, sequenceid=368, compaction requested=false 2024-11-19T17:14:00,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:00,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:00,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-19T17:14:00,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-19T17:14:00,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:14:00,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T17:14:00,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:14:00,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:00,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:14:00,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:00,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:14:00,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:00,118 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-19T17:14:00,119 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3380 sec 2024-11-19T17:14:00,122 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.3530 sec 2024-11-19T17:14:00,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/2dfc74eac7f44c918d43b084b7cc9db5 is 50, key is test_row_0/A:col10/1732036439784/Put/seqid=0 2024-11-19T17:14:00,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741920_1096 (size=12301) 2024-11-19T17:14:00,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/2dfc74eac7f44c918d43b084b7cc9db5 2024-11-19T17:14:00,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/a70908e523984b6ca423bae51f9b3064 is 50, key is test_row_0/B:col10/1732036439784/Put/seqid=0 2024-11-19T17:14:00,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:00,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036500148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:00,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036500151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:00,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:00,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:00,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036500154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:00,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:00,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741921_1097 (size=12301) 2024-11-19T17:14:00,164 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/a70908e523984b6ca423bae51f9b3064 2024-11-19T17:14:00,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/d3b79abd5ab94cf7a475cf741342d458 is 50, key is test_row_0/C:col10/1732036439784/Put/seqid=0 2024-11-19T17:14:00,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741922_1098 (size=12301) 2024-11-19T17:14:00,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/d3b79abd5ab94cf7a475cf741342d458 2024-11-19T17:14:00,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/2dfc74eac7f44c918d43b084b7cc9db5 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/2dfc74eac7f44c918d43b084b7cc9db5 2024-11-19T17:14:00,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/2dfc74eac7f44c918d43b084b7cc9db5, entries=150, sequenceid=386, filesize=12.0 K 2024-11-19T17:14:00,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/a70908e523984b6ca423bae51f9b3064 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/a70908e523984b6ca423bae51f9b3064 2024-11-19T17:14:00,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/a70908e523984b6ca423bae51f9b3064, entries=150, sequenceid=386, filesize=12.0 K 2024-11-19T17:14:00,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/d3b79abd5ab94cf7a475cf741342d458 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/d3b79abd5ab94cf7a475cf741342d458 2024-11-19T17:14:00,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/d3b79abd5ab94cf7a475cf741342d458, entries=150, sequenceid=386, filesize=12.0 K 2024-11-19T17:14:00,230 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 689491498ff99efc87002cc59bc70ef3 in 113ms, sequenceid=386, compaction requested=true 2024-11-19T17:14:00,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:00,230 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:00,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:14:00,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:00,231 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:00,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:14:00,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:00,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:14:00,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:00,232 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:00,232 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/A is initiating minor compaction (all files) 2024-11-19T17:14:00,232 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/A in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:00,232 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:00,232 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/afa60b4eae5b4849b42b83d32b47f42b, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/db845d7b4f354ceab7b3fbac65437a91, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/2dfc74eac7f44c918d43b084b7cc9db5] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=36.9 K 2024-11-19T17:14:00,232 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/B is initiating minor compaction (all files) 2024-11-19T17:14:00,232 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/B in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:00,232 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/105f79f2123f4f99952b16acf9e92fab, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/778389b0eee74594b8f0c0f2fd52ce3f, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/a70908e523984b6ca423bae51f9b3064] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=36.9 K 2024-11-19T17:14:00,233 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 105f79f2123f4f99952b16acf9e92fab, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732036438427 2024-11-19T17:14:00,234 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting afa60b4eae5b4849b42b83d32b47f42b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732036438427 2024-11-19T17:14:00,235 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting db845d7b4f354ceab7b3fbac65437a91, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732036438597 2024-11-19T17:14:00,235 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 778389b0eee74594b8f0c0f2fd52ce3f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732036438597 2024-11-19T17:14:00,235 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2dfc74eac7f44c918d43b084b7cc9db5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1732036439784 2024-11-19T17:14:00,236 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting a70908e523984b6ca423bae51f9b3064, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1732036439784 2024-11-19T17:14:00,252 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#B#compaction#86 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:00,252 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#A#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:00,253 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/98186ecaa2234dfcb8128ab844de1455 is 50, key is test_row_0/B:col10/1732036439784/Put/seqid=0 2024-11-19T17:14:00,253 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/c1952dcf708f4328a4714bc06aefcd3d is 50, key is test_row_0/A:col10/1732036439784/Put/seqid=0 2024-11-19T17:14:00,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:14:00,271 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T17:14:00,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:14:00,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:00,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:14:00,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:00,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:14:00,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:00,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/be36aab5a89b48c1ac0b4c535e877f5a is 50, key is test_row_0/A:col10/1732036440145/Put/seqid=0 2024-11-19T17:14:00,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:00,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036500296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:00,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:00,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036500298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:00,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:00,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036500303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:00,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:00,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:00,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:00,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741923_1099 (size=13289) 2024-11-19T17:14:00,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741924_1100 (size=13289) 2024-11-19T17:14:00,322 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/c1952dcf708f4328a4714bc06aefcd3d as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/c1952dcf708f4328a4714bc06aefcd3d 2024-11-19T17:14:00,333 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/A of 689491498ff99efc87002cc59bc70ef3 into c1952dcf708f4328a4714bc06aefcd3d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:00,333 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:00,333 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/A, priority=13, startTime=1732036440230; duration=0sec 2024-11-19T17:14:00,334 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:00,334 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:A 2024-11-19T17:14:00,334 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:00,335 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:00,335 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/C is initiating minor compaction (all files) 2024-11-19T17:14:00,335 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/C in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:00,336 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/34d9471bddd84730ac1f342ef5a219ed, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/03e2261148df447db2e791124dce802c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/d3b79abd5ab94cf7a475cf741342d458] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=36.9 K 2024-11-19T17:14:00,336 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34d9471bddd84730ac1f342ef5a219ed, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732036438427 2024-11-19T17:14:00,337 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03e2261148df447db2e791124dce802c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732036438597 2024-11-19T17:14:00,337 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3b79abd5ab94cf7a475cf741342d458, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1732036439784 2024-11-19T17:14:00,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741925_1101 (size=17181) 2024-11-19T17:14:00,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/be36aab5a89b48c1ac0b4c535e877f5a 2024-11-19T17:14:00,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/faa0c1ef838242d69cad78cdedf215f8 is 50, key is test_row_0/B:col10/1732036440145/Put/seqid=0 2024-11-19T17:14:00,355 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#C#compaction#89 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:00,356 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/4d070322eac742eb83a3ecd3a8423c6a is 50, key is test_row_0/C:col10/1732036439784/Put/seqid=0 2024-11-19T17:14:00,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741926_1102 (size=12301) 2024-11-19T17:14:00,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/faa0c1ef838242d69cad78cdedf215f8 2024-11-19T17:14:00,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741927_1103 (size=13255) 2024-11-19T17:14:00,402 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/4d070322eac742eb83a3ecd3a8423c6a as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/4d070322eac742eb83a3ecd3a8423c6a 2024-11-19T17:14:00,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/c0bb45ca494c414fa1ad3f6dcda78422 is 50, key is test_row_0/C:col10/1732036440145/Put/seqid=0 2024-11-19T17:14:00,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741928_1104 (size=12301) 2024-11-19T17:14:00,416 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/C of 689491498ff99efc87002cc59bc70ef3 into 4d070322eac742eb83a3ecd3a8423c6a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:00,416 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:00,417 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/C, priority=13, startTime=1732036440231; duration=0sec 2024-11-19T17:14:00,417 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:00,417 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:C 2024-11-19T17:14:00,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036500414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:00,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:00,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036500415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:00,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:00,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:00,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036500415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:00,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:00,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036500624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:00,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:00,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:00,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036500635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:00,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:00,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036500635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:00,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:00,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:00,723 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/98186ecaa2234dfcb8128ab844de1455 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/98186ecaa2234dfcb8128ab844de1455 2024-11-19T17:14:00,730 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/B of 689491498ff99efc87002cc59bc70ef3 into 98186ecaa2234dfcb8128ab844de1455(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:00,731 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:00,731 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/B, priority=13, startTime=1732036440231; duration=0sec 2024-11-19T17:14:00,731 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:00,731 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:B 2024-11-19T17:14:00,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:00,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036500804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:00,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:00,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036500814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:00,813 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_0' in 689491498ff99efc87002cc59bc70ef3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:14:00,815 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_1' in 689491498ff99efc87002cc59bc70ef3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:14:00,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/c0bb45ca494c414fa1ad3f6dcda78422 2024-11-19T17:14:00,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:00,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:00,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/be36aab5a89b48c1ac0b4c535e877f5a as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/be36aab5a89b48c1ac0b4c535e877f5a 2024-11-19T17:14:00,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/be36aab5a89b48c1ac0b4c535e877f5a, entries=250, sequenceid=405, filesize=16.8 K 2024-11-19T17:14:00,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/faa0c1ef838242d69cad78cdedf215f8 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/faa0c1ef838242d69cad78cdedf215f8 2024-11-19T17:14:00,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/faa0c1ef838242d69cad78cdedf215f8, entries=150, sequenceid=405, filesize=12.0 K 2024-11-19T17:14:00,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/c0bb45ca494c414fa1ad3f6dcda78422 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/c0bb45ca494c414fa1ad3f6dcda78422 2024-11-19T17:14:00,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/c0bb45ca494c414fa1ad3f6dcda78422, entries=150, sequenceid=405, filesize=12.0 K 2024-11-19T17:14:00,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 689491498ff99efc87002cc59bc70ef3 in 583ms, sequenceid=405, compaction requested=false 2024-11-19T17:14:00,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:00,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T17:14:00,925 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:14:00,925 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:14:00,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-19T17:14:00,928 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:14:00,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T17:14:00,929 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:14:00,930 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:14:00,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:14:00,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T17:14:00,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:14:00,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:00,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:14:00,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:00,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:14:00,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:00,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/feb0f0467cb74cee93352825c25e9bde is 50, key is test_row_0/A:col10/1732036440297/Put/seqid=0 2024-11-19T17:14:00,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741929_1105 (size=14741) 2024-11-19T17:14:00,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/feb0f0467cb74cee93352825c25e9bde 2024-11-19T17:14:00,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/21b9a06be294438582c16a8e671fb8f4 is 50, key is test_row_0/B:col10/1732036440297/Put/seqid=0 2024-11-19T17:14:00,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:00,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036500980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:00,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:00,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:00,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036500988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:00,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:00,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:00,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036500990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:00,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:00,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:01,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741930_1106 (size=12301) 2024-11-19T17:14:01,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/21b9a06be294438582c16a8e671fb8f4 2024-11-19T17:14:01,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/a8f6a14d1996447d88c8ee5fd930ebda is 50, key is test_row_0/C:col10/1732036440297/Put/seqid=0 2024-11-19T17:14:01,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T17:14:01,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741931_1107 (size=12301) 2024-11-19T17:14:01,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/a8f6a14d1996447d88c8ee5fd930ebda 2024-11-19T17:14:01,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/feb0f0467cb74cee93352825c25e9bde as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/feb0f0467cb74cee93352825c25e9bde 2024-11-19T17:14:01,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/feb0f0467cb74cee93352825c25e9bde, entries=200, sequenceid=422, filesize=14.4 K 2024-11-19T17:14:01,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/21b9a06be294438582c16a8e671fb8f4 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/21b9a06be294438582c16a8e671fb8f4 2024-11-19T17:14:01,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/21b9a06be294438582c16a8e671fb8f4, entries=150, sequenceid=422, filesize=12.0 K 2024-11-19T17:14:01,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/a8f6a14d1996447d88c8ee5fd930ebda as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/a8f6a14d1996447d88c8ee5fd930ebda 2024-11-19T17:14:01,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/a8f6a14d1996447d88c8ee5fd930ebda, entries=150, sequenceid=422, filesize=12.0 K 2024-11-19T17:14:01,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 689491498ff99efc87002cc59bc70ef3 in 132ms, sequenceid=422, compaction requested=true 2024-11-19T17:14:01,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:01,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:14:01,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:01,069 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:01,069 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:01,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:14:01,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:01,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:14:01,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:01,070 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:01,070 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/B is initiating minor compaction (all files) 2024-11-19T17:14:01,070 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/B in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:01,070 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/98186ecaa2234dfcb8128ab844de1455, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/faa0c1ef838242d69cad78cdedf215f8, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/21b9a06be294438582c16a8e671fb8f4] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=37.0 K 2024-11-19T17:14:01,071 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45211 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:01,071 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 98186ecaa2234dfcb8128ab844de1455, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1732036439784 2024-11-19T17:14:01,071 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/A is initiating minor compaction (all files) 2024-11-19T17:14:01,071 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/A in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:01,071 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/c1952dcf708f4328a4714bc06aefcd3d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/be36aab5a89b48c1ac0b4c535e877f5a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/feb0f0467cb74cee93352825c25e9bde] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=44.2 K 2024-11-19T17:14:01,071 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting faa0c1ef838242d69cad78cdedf215f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732036440145 2024-11-19T17:14:01,071 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1952dcf708f4328a4714bc06aefcd3d, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1732036439784 2024-11-19T17:14:01,072 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 21b9a06be294438582c16a8e671fb8f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1732036440295 2024-11-19T17:14:01,072 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting be36aab5a89b48c1ac0b4c535e877f5a, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732036440118 2024-11-19T17:14:01,073 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting feb0f0467cb74cee93352825c25e9bde, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1732036440290 2024-11-19T17:14:01,084 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#B#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:01,084 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/59a1baac523e46deb117d25bab3da0c5 is 50, key is test_row_0/B:col10/1732036440297/Put/seqid=0 2024-11-19T17:14:01,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T17:14:01,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:01,086 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T17:14:01,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:14:01,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:01,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:14:01,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:01,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:14:01,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:01,087 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#A#compaction#96 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:01,088 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/02ffecb125f34fe0a938a11b9a9eb751 is 50, key is test_row_0/A:col10/1732036440297/Put/seqid=0 2024-11-19T17:14:01,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:14:01,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:14:01,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/6c3445ecc48947b7ba1f57518f2a85c4 is 50, key is test_row_0/A:col10/1732036440976/Put/seqid=0 2024-11-19T17:14:01,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741932_1108 (size=13391) 2024-11-19T17:14:01,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741933_1109 (size=13391) 2024-11-19T17:14:01,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:01,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036501118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:01,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:01,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:01,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:01,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036501119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:01,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:01,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:01,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036501121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:01,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:01,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:01,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:01,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:01,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:01,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:01,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741934_1110 (size=12301) 2024-11-19T17:14:01,139 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/6c3445ecc48947b7ba1f57518f2a85c4 2024-11-19T17:14:01,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/0187133a8e004438a281a70320cdcb12 is 50, key is test_row_0/B:col10/1732036440976/Put/seqid=0 2024-11-19T17:14:01,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741935_1111 (size=12301) 2024-11-19T17:14:01,154 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/0187133a8e004438a281a70320cdcb12 2024-11-19T17:14:01,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/3946624d13164aff9de7abd136999c7f is 50, key is test_row_0/C:col10/1732036440976/Put/seqid=0 2024-11-19T17:14:01,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741936_1112 (size=12301) 2024-11-19T17:14:01,171 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/3946624d13164aff9de7abd136999c7f 2024-11-19T17:14:01,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/6c3445ecc48947b7ba1f57518f2a85c4 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/6c3445ecc48947b7ba1f57518f2a85c4 2024-11-19T17:14:01,188 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/6c3445ecc48947b7ba1f57518f2a85c4, entries=150, sequenceid=441, filesize=12.0 K 2024-11-19T17:14:01,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/0187133a8e004438a281a70320cdcb12 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/0187133a8e004438a281a70320cdcb12 2024-11-19T17:14:01,197 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/0187133a8e004438a281a70320cdcb12, entries=150, sequenceid=441, filesize=12.0 K 2024-11-19T17:14:01,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/3946624d13164aff9de7abd136999c7f as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/3946624d13164aff9de7abd136999c7f 2024-11-19T17:14:01,204 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/3946624d13164aff9de7abd136999c7f, entries=150, sequenceid=441, filesize=12.0 K 2024-11-19T17:14:01,205 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 689491498ff99efc87002cc59bc70ef3 in 119ms, sequenceid=441, compaction requested=true 2024-11-19T17:14:01,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:01,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:01,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-19T17:14:01,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-19T17:14:01,210 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-19T17:14:01,210 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 278 msec 2024-11-19T17:14:01,212 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 285 msec 2024-11-19T17:14:01,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:14:01,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T17:14:01,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:14:01,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:01,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:14:01,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:01,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:14:01,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T17:14:01,244 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:14:01,245 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:14:01,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-19T17:14:01,248 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:14:01,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T17:14:01,249 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:14:01,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/7751205b907c44648258e768d54d3a50 is 50, key is test_row_0/A:col10/1732036441118/Put/seqid=0 2024-11-19T17:14:01,250 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:14:01,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741937_1113 (size=14741) 2024-11-19T17:14:01,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036501282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:01,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:01,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:01,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:01,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036501286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:01,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036501290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:01,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:01,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:01,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:01,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:01,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:01,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:01,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T17:14:01,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:01,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036501394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:01,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:01,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:01,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:01,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T17:14:01,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:01,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:14:01,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:01,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:01,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:01,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:01,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036501405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:01,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036501405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:01,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:01,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:01,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:01,537 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/59a1baac523e46deb117d25bab3da0c5 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/59a1baac523e46deb117d25bab3da0c5 2024-11-19T17:14:01,541 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/02ffecb125f34fe0a938a11b9a9eb751 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/02ffecb125f34fe0a938a11b9a9eb751 2024-11-19T17:14:01,548 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/B of 689491498ff99efc87002cc59bc70ef3 into 59a1baac523e46deb117d25bab3da0c5(size=13.1 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:01,548 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:01,549 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/B, priority=13, startTime=1732036441068; duration=0sec 2024-11-19T17:14:01,549 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:01,549 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:B 2024-11-19T17:14:01,549 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:14:01,551 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/A of 689491498ff99efc87002cc59bc70ef3 into 02ffecb125f34fe0a938a11b9a9eb751(size=13.1 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:01,551 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:01,551 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/A, priority=13, startTime=1732036441068; duration=0sec 2024-11-19T17:14:01,552 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:01,552 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:14:01,552 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:A 2024-11-19T17:14:01,552 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/C is initiating minor compaction (all files) 2024-11-19T17:14:01,552 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/C in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:01,552 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/4d070322eac742eb83a3ecd3a8423c6a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/c0bb45ca494c414fa1ad3f6dcda78422, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/a8f6a14d1996447d88c8ee5fd930ebda, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/3946624d13164aff9de7abd136999c7f] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=49.0 K 2024-11-19T17:14:01,553 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d070322eac742eb83a3ecd3a8423c6a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1732036439784 2024-11-19T17:14:01,554 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting c0bb45ca494c414fa1ad3f6dcda78422, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732036440145 2024-11-19T17:14:01,554 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting a8f6a14d1996447d88c8ee5fd930ebda, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1732036440295 2024-11-19T17:14:01,555 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 3946624d13164aff9de7abd136999c7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1732036440976 2024-11-19T17:14:01,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T17:14:01,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:01,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:14:01,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:01,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:01,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T17:14:01,583 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#C#compaction#101 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:01,584 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/a31a066021b34294b1929d67671cf008 is 50, key is test_row_0/C:col10/1732036440976/Put/seqid=0 2024-11-19T17:14:01,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741938_1114 (size=13391) 2024-11-19T17:14:01,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036501604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:01,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:01,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:01,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:01,618 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/a31a066021b34294b1929d67671cf008 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/a31a066021b34294b1929d67671cf008 2024-11-19T17:14:01,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036501617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:01,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:01,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:01,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:01,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:01,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036501617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:01,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:01,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:01,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:01,629 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/C of 689491498ff99efc87002cc59bc70ef3 into a31a066021b34294b1929d67671cf008(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:01,629 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:01,629 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/C, priority=12, startTime=1732036441069; duration=0sec 2024-11-19T17:14:01,629 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:01,629 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:C 2024-11-19T17:14:01,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/7751205b907c44648258e768d54d3a50 2024-11-19T17:14:01,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/98937ed889014f0a96e09886b39dfb50 is 50, key is test_row_0/B:col10/1732036441118/Put/seqid=0 2024-11-19T17:14:01,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741939_1115 (size=12301) 2024-11-19T17:14:01,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T17:14:01,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:01,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:14:01,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:01,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:01,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:01,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T17:14:01,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:01,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:14:01,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:01,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:01,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:01,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:01,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T17:14:01,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:01,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036501914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:01,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:01,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:01,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:01,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:01,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036501934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:01,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:01,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036501935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:02,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T17:14:02,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:02,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:14:02,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:02,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:02,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:02,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:02,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/98937ed889014f0a96e09886b39dfb50 2024-11-19T17:14:02,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/fdb799d488884069931f3e68489f59ce is 50, key is test_row_0/C:col10/1732036441118/Put/seqid=0 2024-11-19T17:14:02,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741940_1116 (size=12301) 2024-11-19T17:14:02,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T17:14:02,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:02,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:14:02,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:02,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:02,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:02,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:02,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T17:14:02,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:02,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:14:02,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:02,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:02,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:02,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:02,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T17:14:02,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:02,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036502433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:02,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:02,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:02,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:02,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:02,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036502444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:02,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:02,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:02,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:02,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:02,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036502454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:02,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:02,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:02,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:02,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T17:14:02,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:02,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:14:02,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:02,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:02,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:02,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:02,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/fdb799d488884069931f3e68489f59ce 2024-11-19T17:14:02,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/7751205b907c44648258e768d54d3a50 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/7751205b907c44648258e768d54d3a50 2024-11-19T17:14:02,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/7751205b907c44648258e768d54d3a50, entries=200, sequenceid=455, filesize=14.4 K 2024-11-19T17:14:02,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/98937ed889014f0a96e09886b39dfb50 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/98937ed889014f0a96e09886b39dfb50 2024-11-19T17:14:02,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/98937ed889014f0a96e09886b39dfb50, entries=150, sequenceid=455, filesize=12.0 K 2024-11-19T17:14:02,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/fdb799d488884069931f3e68489f59ce as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/fdb799d488884069931f3e68489f59ce 2024-11-19T17:14:02,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/fdb799d488884069931f3e68489f59ce, entries=150, sequenceid=455, filesize=12.0 K 2024-11-19T17:14:02,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 689491498ff99efc87002cc59bc70ef3 in 1329ms, sequenceid=455, compaction requested=true 2024-11-19T17:14:02,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:02,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:14:02,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:02,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:14:02,568 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:02,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:02,569 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:02,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:14:02,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:02,570 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:02,570 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/A is initiating minor compaction (all files) 2024-11-19T17:14:02,570 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:02,570 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/A in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:02,570 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/B is initiating minor compaction (all files) 2024-11-19T17:14:02,571 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/02ffecb125f34fe0a938a11b9a9eb751, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/6c3445ecc48947b7ba1f57518f2a85c4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/7751205b907c44648258e768d54d3a50] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=39.5 K 2024-11-19T17:14:02,571 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/B in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:02,571 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/59a1baac523e46deb117d25bab3da0c5, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/0187133a8e004438a281a70320cdcb12, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/98937ed889014f0a96e09886b39dfb50] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=37.1 K 2024-11-19T17:14:02,573 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02ffecb125f34fe0a938a11b9a9eb751, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1732036440295 2024-11-19T17:14:02,574 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 59a1baac523e46deb117d25bab3da0c5, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1732036440295 2024-11-19T17:14:02,574 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c3445ecc48947b7ba1f57518f2a85c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1732036440976 2024-11-19T17:14:02,574 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 0187133a8e004438a281a70320cdcb12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1732036440976 2024-11-19T17:14:02,575 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7751205b907c44648258e768d54d3a50, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1732036441115 2024-11-19T17:14:02,575 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 98937ed889014f0a96e09886b39dfb50, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1732036441117 2024-11-19T17:14:02,595 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#A#compaction#104 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:02,596 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/c97b632038b848c0a6014cf8c8a379c3 is 50, key is test_row_0/A:col10/1732036441118/Put/seqid=0 2024-11-19T17:14:02,598 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#B#compaction#105 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:02,598 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/02f94b05301c4606ae362c17470dcf79 is 50, key is test_row_0/B:col10/1732036441118/Put/seqid=0 2024-11-19T17:14:02,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741942_1118 (size=13493) 2024-11-19T17:14:02,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741941_1117 (size=13493) 2024-11-19T17:14:02,642 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/c97b632038b848c0a6014cf8c8a379c3 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/c97b632038b848c0a6014cf8c8a379c3 2024-11-19T17:14:02,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T17:14:02,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:02,647 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T17:14:02,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:14:02,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:02,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:14:02,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:02,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:14:02,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:02,651 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/A of 689491498ff99efc87002cc59bc70ef3 into c97b632038b848c0a6014cf8c8a379c3(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:02,652 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:02,652 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/A, priority=13, startTime=1732036442568; duration=0sec 2024-11-19T17:14:02,652 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:02,652 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:A 2024-11-19T17:14:02,652 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-19T17:14:02,653 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-19T17:14:02,653 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-19T17:14:02,653 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. because compaction request was cancelled 2024-11-19T17:14:02,653 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:C 2024-11-19T17:14:02,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/3d730d95ce6f4a30b037621edde132f3 is 50, key is test_row_0/A:col10/1732036441275/Put/seqid=0 2024-11-19T17:14:02,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741943_1119 (size=12301) 2024-11-19T17:14:02,681 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/3d730d95ce6f4a30b037621edde132f3 2024-11-19T17:14:02,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/96dcf8d0294f4a1d8255fa3551ee5627 is 50, key is test_row_0/B:col10/1732036441275/Put/seqid=0 2024-11-19T17:14:02,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741944_1120 (size=12301) 2024-11-19T17:14:02,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:14:02,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:14:02,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:02,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036502873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:02,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:02,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:02,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:02,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:02,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036502884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:02,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:02,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:02,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:02,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036502984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:02,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:02,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:02,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:02,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:03,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036502996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:03,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:03,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:03,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:03,035 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/02f94b05301c4606ae362c17470dcf79 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/02f94b05301c4606ae362c17470dcf79 2024-11-19T17:14:03,044 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/B of 689491498ff99efc87002cc59bc70ef3 into 02f94b05301c4606ae362c17470dcf79(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:03,044 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:03,044 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/B, priority=13, startTime=1732036442568; duration=0sec 2024-11-19T17:14:03,044 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:03,046 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:B 2024-11-19T17:14:03,111 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/96dcf8d0294f4a1d8255fa3551ee5627 2024-11-19T17:14:03,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/89f67b4d082d4c5ba5c0163bf8cb12ce is 50, key is test_row_0/C:col10/1732036441275/Put/seqid=0 2024-11-19T17:14:03,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741945_1121 (size=12301) 2024-11-19T17:14:03,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:03,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036503193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:03,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:03,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:03,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:03,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:03,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036503204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:03,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:03,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:03,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:03,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T17:14:03,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:03,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036503444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:03,447 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_1' in 689491498ff99efc87002cc59bc70ef3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:14:03,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:03,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:03,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:03,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036503464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:03,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:03,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036503464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:03,470 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_1' in 689491498ff99efc87002cc59bc70ef3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:14:03,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:03,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:03,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:03,470 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_1' in 689491498ff99efc87002cc59bc70ef3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:14:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:03,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036503504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:03,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:03,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:03,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:03,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:03,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036503514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:03,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:03,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:03,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:03,534 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/89f67b4d082d4c5ba5c0163bf8cb12ce 2024-11-19T17:14:03,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/3d730d95ce6f4a30b037621edde132f3 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/3d730d95ce6f4a30b037621edde132f3 2024-11-19T17:14:03,560 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/3d730d95ce6f4a30b037621edde132f3, entries=150, sequenceid=476, filesize=12.0 K 2024-11-19T17:14:03,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/96dcf8d0294f4a1d8255fa3551ee5627 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/96dcf8d0294f4a1d8255fa3551ee5627 2024-11-19T17:14:03,574 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/96dcf8d0294f4a1d8255fa3551ee5627, entries=150, sequenceid=476, filesize=12.0 K 2024-11-19T17:14:03,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/89f67b4d082d4c5ba5c0163bf8cb12ce as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/89f67b4d082d4c5ba5c0163bf8cb12ce 2024-11-19T17:14:03,582 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/89f67b4d082d4c5ba5c0163bf8cb12ce, entries=150, sequenceid=476, filesize=12.0 K 2024-11-19T17:14:03,594 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 689491498ff99efc87002cc59bc70ef3 in 948ms, sequenceid=476, compaction requested=true 2024-11-19T17:14:03,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:03,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:03,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-19T17:14:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-19T17:14:03,599 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-19T17:14:03,599 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3470 sec 2024-11-19T17:14:03,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 2.3570 sec 2024-11-19T17:14:04,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:14:04,017 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T17:14:04,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:14:04,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:04,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:14:04,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:04,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:14:04,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:04,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/ad484189020047bc9b50a6caed1555bf is 50, key is test_row_0/A:col10/1732036442877/Put/seqid=0 2024-11-19T17:14:04,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741946_1122 (size=12301) 2024-11-19T17:14:04,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/ad484189020047bc9b50a6caed1555bf 2024-11-19T17:14:04,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/68762490557546de99d2ae4ab37398e0 is 50, key is test_row_0/B:col10/1732036442877/Put/seqid=0 2024-11-19T17:14:04,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:04,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036504080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:04,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:04,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036504081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:04,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:04,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:04,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:04,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:04,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:04,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:04,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741947_1123 (size=12301) 2024-11-19T17:14:04,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/68762490557546de99d2ae4ab37398e0 2024-11-19T17:14:04,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/599d715747f140cea68cc837063930bd is 50, key is test_row_0/C:col10/1732036442877/Put/seqid=0 2024-11-19T17:14:04,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741948_1124 (size=12301) 2024-11-19T17:14:04,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/599d715747f140cea68cc837063930bd 2024-11-19T17:14:04,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/ad484189020047bc9b50a6caed1555bf as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/ad484189020047bc9b50a6caed1555bf 2024-11-19T17:14:04,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:04,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036504194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:04,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:04,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:04,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:04,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:04,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036504194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:04,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:04,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:04,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:04,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/ad484189020047bc9b50a6caed1555bf, entries=150, sequenceid=493, filesize=12.0 K 2024-11-19T17:14:04,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/68762490557546de99d2ae4ab37398e0 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/68762490557546de99d2ae4ab37398e0 2024-11-19T17:14:04,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/68762490557546de99d2ae4ab37398e0, entries=150, sequenceid=493, filesize=12.0 K 2024-11-19T17:14:04,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/599d715747f140cea68cc837063930bd as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/599d715747f140cea68cc837063930bd 2024-11-19T17:14:04,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/599d715747f140cea68cc837063930bd, entries=150, sequenceid=493, filesize=12.0 K 2024-11-19T17:14:04,232 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 689491498ff99efc87002cc59bc70ef3 in 215ms, sequenceid=493, compaction requested=true 2024-11-19T17:14:04,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:04,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:14:04,233 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:04,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:04,233 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:04,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:14:04,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:04,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:14:04,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:04,236 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:04,236 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:04,236 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/A is initiating minor compaction (all files) 2024-11-19T17:14:04,236 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/B is initiating minor compaction (all files) 2024-11-19T17:14:04,236 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/A in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:04,236 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/B in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:04,236 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/c97b632038b848c0a6014cf8c8a379c3, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/3d730d95ce6f4a30b037621edde132f3, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/ad484189020047bc9b50a6caed1555bf] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=37.2 K 2024-11-19T17:14:04,236 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/02f94b05301c4606ae362c17470dcf79, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/96dcf8d0294f4a1d8255fa3551ee5627, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/68762490557546de99d2ae4ab37398e0] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=37.2 K 2024-11-19T17:14:04,237 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting c97b632038b848c0a6014cf8c8a379c3, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1732036441117 2024-11-19T17:14:04,237 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 02f94b05301c4606ae362c17470dcf79, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1732036441117 2024-11-19T17:14:04,240 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 96dcf8d0294f4a1d8255fa3551ee5627, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1732036441275 2024-11-19T17:14:04,240 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d730d95ce6f4a30b037621edde132f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1732036441275 2024-11-19T17:14:04,241 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 68762490557546de99d2ae4ab37398e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732036442863 2024-11-19T17:14:04,241 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad484189020047bc9b50a6caed1555bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732036442863 2024-11-19T17:14:04,254 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#B#compaction#112 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:04,255 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/cdba7d8e50b74ed5abd8833dc16bef15 is 50, key is test_row_0/B:col10/1732036442877/Put/seqid=0 2024-11-19T17:14:04,258 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#A#compaction#113 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:04,259 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/1fb27786b9354178929d60598bdcf52f is 50, key is test_row_0/A:col10/1732036442877/Put/seqid=0 2024-11-19T17:14:04,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741950_1126 (size=13595) 2024-11-19T17:14:04,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741949_1125 (size=13595) 2024-11-19T17:14:04,289 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/cdba7d8e50b74ed5abd8833dc16bef15 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/cdba7d8e50b74ed5abd8833dc16bef15 2024-11-19T17:14:04,301 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/1fb27786b9354178929d60598bdcf52f as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/1fb27786b9354178929d60598bdcf52f 2024-11-19T17:14:04,303 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/B of 689491498ff99efc87002cc59bc70ef3 into cdba7d8e50b74ed5abd8833dc16bef15(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:04,303 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:04,303 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/B, priority=13, startTime=1732036444233; duration=0sec 2024-11-19T17:14:04,303 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:04,303 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:B 2024-11-19T17:14:04,303 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:14:04,306 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50294 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:14:04,306 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/C is initiating minor compaction (all files) 2024-11-19T17:14:04,306 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/C in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:04,306 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/a31a066021b34294b1929d67671cf008, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/fdb799d488884069931f3e68489f59ce, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/89f67b4d082d4c5ba5c0163bf8cb12ce, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/599d715747f140cea68cc837063930bd] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=49.1 K 2024-11-19T17:14:04,307 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting a31a066021b34294b1929d67671cf008, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1732036440976 2024-11-19T17:14:04,307 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting fdb799d488884069931f3e68489f59ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1732036441117 2024-11-19T17:14:04,308 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 89f67b4d082d4c5ba5c0163bf8cb12ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1732036441275 2024-11-19T17:14:04,309 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/A of 689491498ff99efc87002cc59bc70ef3 into 1fb27786b9354178929d60598bdcf52f(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:04,309 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:04,309 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/A, priority=13, startTime=1732036444232; duration=0sec 2024-11-19T17:14:04,309 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:04,309 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:A 2024-11-19T17:14:04,311 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 599d715747f140cea68cc837063930bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732036442863 2024-11-19T17:14:04,327 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#C#compaction#114 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:04,328 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/464cb1d766404842b15ea4c183019a0f is 50, key is test_row_0/C:col10/1732036442877/Put/seqid=0 2024-11-19T17:14:04,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741951_1127 (size=13527) 2024-11-19T17:14:04,352 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/464cb1d766404842b15ea4c183019a0f as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/464cb1d766404842b15ea4c183019a0f 2024-11-19T17:14:04,375 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/C of 689491498ff99efc87002cc59bc70ef3 into 464cb1d766404842b15ea4c183019a0f(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:04,375 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:04,376 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/C, priority=12, startTime=1732036444235; duration=0sec 2024-11-19T17:14:04,376 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:04,376 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:C 2024-11-19T17:14:04,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:14:04,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T17:14:04,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:14:04,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:04,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:14:04,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:04,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:14:04,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:04,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/b94d53c1e17644cbbdbbd371d0cd5797 is 50, key is test_row_0/A:col10/1732036444077/Put/seqid=0 2024-11-19T17:14:04,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741952_1128 (size=14741) 2024-11-19T17:14:04,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:04,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036504462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:04,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:04,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:04,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/b94d53c1e17644cbbdbbd371d0cd5797 2024-11-19T17:14:04,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:04,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:04,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036504464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:04,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:04,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:04,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:04,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/d670c5cf48ca4308bcf95266b5a968b3 is 50, key is test_row_0/B:col10/1732036444077/Put/seqid=0 2024-11-19T17:14:04,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741953_1129 (size=12301) 2024-11-19T17:14:04,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:04,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036504574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:04,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:04,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036504574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:04,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:04,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:04,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:04,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:04,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:04,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:04,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:04,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036504784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:04,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:04,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036504784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:04,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:04,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:04,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:04,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:04,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:04,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:04,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/d670c5cf48ca4308bcf95266b5a968b3 2024-11-19T17:14:04,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/b8783eeb558b4c7e930fa136023f9fa2 is 50, key is test_row_0/C:col10/1732036444077/Put/seqid=0 2024-11-19T17:14:05,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741954_1130 (size=12301) 2024-11-19T17:14:05,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/b8783eeb558b4c7e930fa136023f9fa2 2024-11-19T17:14:05,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/b94d53c1e17644cbbdbbd371d0cd5797 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/b94d53c1e17644cbbdbbd371d0cd5797 2024-11-19T17:14:05,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/b94d53c1e17644cbbdbbd371d0cd5797, entries=200, sequenceid=515, filesize=14.4 K 2024-11-19T17:14:05,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/d670c5cf48ca4308bcf95266b5a968b3 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/d670c5cf48ca4308bcf95266b5a968b3 2024-11-19T17:14:05,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/d670c5cf48ca4308bcf95266b5a968b3, entries=150, sequenceid=515, filesize=12.0 K 2024-11-19T17:14:05,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/b8783eeb558b4c7e930fa136023f9fa2 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/b8783eeb558b4c7e930fa136023f9fa2 2024-11-19T17:14:05,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/b8783eeb558b4c7e930fa136023f9fa2, entries=150, sequenceid=515, filesize=12.0 K 2024-11-19T17:14:05,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 689491498ff99efc87002cc59bc70ef3 in 648ms, sequenceid=515, compaction requested=false 2024-11-19T17:14:05,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:05,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:14:05,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T17:14:05,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:14:05,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:05,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:14:05,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:05,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:14:05,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:05,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/f04b573608f74230a4fece29df39178c is 50, key is test_row_0/A:col10/1732036445098/Put/seqid=0 2024-11-19T17:14:05,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741955_1131 (size=17181) 2024-11-19T17:14:05,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/f04b573608f74230a4fece29df39178c 2024-11-19T17:14:05,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:05,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036505143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:05,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:05,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:05,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:05,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:05,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036505149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:05,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:05,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/147abe740d9e42fa8fa8dfcabe93b159 is 50, key is test_row_0/B:col10/1732036445098/Put/seqid=0 2024-11-19T17:14:05,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741956_1132 (size=12301) 2024-11-19T17:14:05,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:05,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036505264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:05,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:05,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036505264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:05,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:05,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:05,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:05,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:05,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:05,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:05,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T17:14:05,405 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:14:05,405 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:14:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-19T17:14:05,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T17:14:05,409 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:14:05,410 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:14:05,410 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:14:05,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:05,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33806 deadline: 1732036505454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:05,460 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_1' in 689491498ff99efc87002cc59bc70ef3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:14:05,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:05,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:05,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:05,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:05,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036505473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:05,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:05,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:05,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:05,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:05,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036505473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:05,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:05,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:05,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:05,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:05,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33820 deadline: 1732036505484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:05,487 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_1' in 689491498ff99efc87002cc59bc70ef3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:14:05,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:05,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:05,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:05,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:05,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33816 deadline: 1732036505485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:05,490 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_1' in 689491498ff99efc87002cc59bc70ef3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:14:05,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:05,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:05,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:05,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T17:14:05,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-19T17:14:05,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:05,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:14:05,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:05,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:05,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:05,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:05,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/147abe740d9e42fa8fa8dfcabe93b159 2024-11-19T17:14:05,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/de22ffa6d3444d7e9bc11a68465d7b94 is 50, key is test_row_0/C:col10/1732036445098/Put/seqid=0 2024-11-19T17:14:05,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741957_1133 (size=12301) 2024-11-19T17:14:05,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-19T17:14:05,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:05,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:14:05,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:05,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:05,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T17:14:05,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:05,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036505785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:05,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:05,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036505785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:05,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:05,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:05,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:05,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:05,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:05,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:05,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-19T17:14:05,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:05,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:14:05,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:05,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:05,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:06,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-19T17:14:06,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:06,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:14:06,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:06,028 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:06,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T17:14:06,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/de22ffa6d3444d7e9bc11a68465d7b94 2024-11-19T17:14:06,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/f04b573608f74230a4fece29df39178c as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/f04b573608f74230a4fece29df39178c 2024-11-19T17:14:06,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/f04b573608f74230a4fece29df39178c, entries=250, sequenceid=530, filesize=16.8 K 2024-11-19T17:14:06,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/147abe740d9e42fa8fa8dfcabe93b159 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/147abe740d9e42fa8fa8dfcabe93b159 2024-11-19T17:14:06,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/147abe740d9e42fa8fa8dfcabe93b159, entries=150, sequenceid=530, filesize=12.0 K 2024-11-19T17:14:06,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/de22ffa6d3444d7e9bc11a68465d7b94 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/de22ffa6d3444d7e9bc11a68465d7b94 2024-11-19T17:14:06,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/de22ffa6d3444d7e9bc11a68465d7b94, entries=150, sequenceid=530, filesize=12.0 K 2024-11-19T17:14:06,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 689491498ff99efc87002cc59bc70ef3 in 967ms, sequenceid=530, compaction requested=true 2024-11-19T17:14:06,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:06,066 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:06,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:14:06,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:06,067 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:06,068 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:06,068 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/A is initiating minor compaction (all files) 2024-11-19T17:14:06,068 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/A in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:06,068 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/1fb27786b9354178929d60598bdcf52f, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/b94d53c1e17644cbbdbbd371d0cd5797, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/f04b573608f74230a4fece29df39178c] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=44.5 K 2024-11-19T17:14:06,069 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:06,069 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/B is initiating minor compaction (all files) 2024-11-19T17:14:06,069 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/B in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:06,069 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/cdba7d8e50b74ed5abd8833dc16bef15, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/d670c5cf48ca4308bcf95266b5a968b3, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/147abe740d9e42fa8fa8dfcabe93b159] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=37.3 K 2024-11-19T17:14:06,069 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fb27786b9354178929d60598bdcf52f, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732036442863 2024-11-19T17:14:06,070 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting cdba7d8e50b74ed5abd8833dc16bef15, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732036442863 2024-11-19T17:14:06,070 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting b94d53c1e17644cbbdbbd371d0cd5797, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=515, earliestPutTs=1732036444070 2024-11-19T17:14:06,071 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting d670c5cf48ca4308bcf95266b5a968b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=515, earliestPutTs=1732036444077 2024-11-19T17:14:06,071 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting f04b573608f74230a4fece29df39178c, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=530, earliestPutTs=1732036444412 2024-11-19T17:14:06,071 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 147abe740d9e42fa8fa8dfcabe93b159, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=530, earliestPutTs=1732036444462 2024-11-19T17:14:06,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:14:06,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:06,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:14:06,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:06,086 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#B#compaction#121 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:06,087 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/05ca54b3660a4026844bd5ab415b557b is 50, key is test_row_0/B:col10/1732036445098/Put/seqid=0 2024-11-19T17:14:06,095 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#A#compaction#122 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:06,096 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/a649ac6064d749ac9c21b51760ecef49 is 50, key is test_row_0/A:col10/1732036445098/Put/seqid=0 2024-11-19T17:14:06,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741958_1134 (size=13697) 2024-11-19T17:14:06,144 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/05ca54b3660a4026844bd5ab415b557b as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/05ca54b3660a4026844bd5ab415b557b 2024-11-19T17:14:06,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741959_1135 (size=13697) 2024-11-19T17:14:06,154 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/B of 689491498ff99efc87002cc59bc70ef3 into 05ca54b3660a4026844bd5ab415b557b(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:06,154 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:06,154 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/B, priority=13, startTime=1732036446067; duration=0sec 2024-11-19T17:14:06,154 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:06,155 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:B 2024-11-19T17:14:06,155 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:06,157 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38129 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:06,157 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/C is initiating minor compaction (all files) 2024-11-19T17:14:06,157 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/C in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:06,158 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/464cb1d766404842b15ea4c183019a0f, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/b8783eeb558b4c7e930fa136023f9fa2, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/de22ffa6d3444d7e9bc11a68465d7b94] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=37.2 K 2024-11-19T17:14:06,158 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 464cb1d766404842b15ea4c183019a0f, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732036442863 2024-11-19T17:14:06,159 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting b8783eeb558b4c7e930fa136023f9fa2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=515, earliestPutTs=1732036444077 2024-11-19T17:14:06,159 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting de22ffa6d3444d7e9bc11a68465d7b94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=530, earliestPutTs=1732036444462 2024-11-19T17:14:06,159 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/a649ac6064d749ac9c21b51760ecef49 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/a649ac6064d749ac9c21b51760ecef49 2024-11-19T17:14:06,167 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/A of 689491498ff99efc87002cc59bc70ef3 into a649ac6064d749ac9c21b51760ecef49(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:06,167 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:06,167 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/A, priority=13, startTime=1732036446066; duration=0sec 2024-11-19T17:14:06,167 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:06,168 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:A 2024-11-19T17:14:06,179 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#C#compaction#123 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:06,180 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/d78b2cb860d444d682e368f9afa93080 is 50, key is test_row_0/C:col10/1732036445098/Put/seqid=0 2024-11-19T17:14:06,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-19T17:14:06,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:06,184 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T17:14:06,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:14:06,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:06,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:14:06,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:06,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:14:06,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:06,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/3bc1afb6ab834bee8fdadcc418159728 is 50, key is test_row_0/A:col10/1732036445140/Put/seqid=0 2024-11-19T17:14:06,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741960_1136 (size=13629) 2024-11-19T17:14:06,221 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/d78b2cb860d444d682e368f9afa93080 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/d78b2cb860d444d682e368f9afa93080 2024-11-19T17:14:06,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741961_1137 (size=12301) 2024-11-19T17:14:06,233 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/3bc1afb6ab834bee8fdadcc418159728 2024-11-19T17:14:06,236 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/C of 689491498ff99efc87002cc59bc70ef3 into d78b2cb860d444d682e368f9afa93080(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:06,236 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:06,236 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/C, priority=13, startTime=1732036446081; duration=0sec 2024-11-19T17:14:06,236 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:06,236 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:C 2024-11-19T17:14:06,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/21e29e1c8c0e4da191c5fd07569aa9bb is 50, key is test_row_0/B:col10/1732036445140/Put/seqid=0 2024-11-19T17:14:06,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741962_1138 (size=12301) 2024-11-19T17:14:06,261 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/21e29e1c8c0e4da191c5fd07569aa9bb 2024-11-19T17:14:06,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/21ad9e57b563402a982fad1237461ad8 is 50, key is test_row_0/C:col10/1732036445140/Put/seqid=0 2024-11-19T17:14:06,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741963_1139 (size=12301) 2024-11-19T17:14:06,286 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/21ad9e57b563402a982fad1237461ad8 2024-11-19T17:14:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:14:06,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:14:06,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/3bc1afb6ab834bee8fdadcc418159728 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/3bc1afb6ab834bee8fdadcc418159728 2024-11-19T17:14:06,323 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/3bc1afb6ab834bee8fdadcc418159728, entries=150, sequenceid=549, filesize=12.0 K 2024-11-19T17:14:06,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/21e29e1c8c0e4da191c5fd07569aa9bb as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/21e29e1c8c0e4da191c5fd07569aa9bb 2024-11-19T17:14:06,331 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/21e29e1c8c0e4da191c5fd07569aa9bb, entries=150, sequenceid=549, filesize=12.0 K 2024-11-19T17:14:06,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/21ad9e57b563402a982fad1237461ad8 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/21ad9e57b563402a982fad1237461ad8 2024-11-19T17:14:06,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036506335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:06,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:06,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:06,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:06,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036506342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:06,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:06,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:06,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:06,351 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/21ad9e57b563402a982fad1237461ad8, entries=150, sequenceid=549, filesize=12.0 K 2024-11-19T17:14:06,352 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 689491498ff99efc87002cc59bc70ef3 in 169ms, sequenceid=549, compaction requested=false 2024-11-19T17:14:06,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:06,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:06,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-19T17:14:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-19T17:14:06,357 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-19T17:14:06,357 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 944 msec 2024-11-19T17:14:06,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 953 msec 2024-11-19T17:14:06,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T17:14:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:14:06,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:14:06,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:06,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:14:06,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:06,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:14:06,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:06,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/de07c1b55de24175acc209dc1a1c3ff6 is 50, key is test_row_0/A:col10/1732036446340/Put/seqid=0 2024-11-19T17:14:06,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036506497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:06,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741964_1140 (size=14741) 2024-11-19T17:14:06,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=567 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/de07c1b55de24175acc209dc1a1c3ff6 2024-11-19T17:14:06,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:06,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036506501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:06,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:06,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:06,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:06,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/c0f519feca0540dda4c78b51c9ad680b is 50, key is test_row_0/B:col10/1732036446340/Put/seqid=0 2024-11-19T17:14:06,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T17:14:06,545 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:14:06,545 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:14:06,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees 2024-11-19T17:14:06,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-19T17:14:06,550 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:14:06,551 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:14:06,551 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:14:06,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741965_1141 (size=12301) 2024-11-19T17:14:06,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036506604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:06,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:06,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:06,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:06,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036506613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:06,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:06,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:06,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-19T17:14:06,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-19T17:14:06,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:06,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:14:06,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:06,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:06,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:06,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:06,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:06,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036506814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:06,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:06,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:06,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:06,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:06,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036506824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:06,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:06,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:06,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:06,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-19T17:14:06,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:06,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:14:06,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:06,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:06,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:06,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-19T17:14:06,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:06,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=567 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/c0f519feca0540dda4c78b51c9ad680b 2024-11-19T17:14:06,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/3ffe66af02894054a0debdf857ef87ca is 50, key is test_row_0/C:col10/1732036446340/Put/seqid=0 2024-11-19T17:14:07,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741966_1142 (size=12301) 2024-11-19T17:14:07,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-19T17:14:07,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:07,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:14:07,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:07,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:07,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:07,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:07,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:07,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036507123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:07,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:07,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:07,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:07,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036507144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:07,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:07,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:07,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:07,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-19T17:14:07,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:07,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:14:07,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:07,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:07,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-19T17:14:07,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-19T17:14:07,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:07,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:14:07,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:07,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:07,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:07,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=567 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/3ffe66af02894054a0debdf857ef87ca 2024-11-19T17:14:07,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/de07c1b55de24175acc209dc1a1c3ff6 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/de07c1b55de24175acc209dc1a1c3ff6 2024-11-19T17:14:07,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/de07c1b55de24175acc209dc1a1c3ff6, entries=200, sequenceid=567, filesize=14.4 K 2024-11-19T17:14:07,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/c0f519feca0540dda4c78b51c9ad680b as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/c0f519feca0540dda4c78b51c9ad680b 2024-11-19T17:14:07,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/c0f519feca0540dda4c78b51c9ad680b, entries=150, sequenceid=567, filesize=12.0 K 2024-11-19T17:14:07,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/3ffe66af02894054a0debdf857ef87ca as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/3ffe66af02894054a0debdf857ef87ca 2024-11-19T17:14:07,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/3ffe66af02894054a0debdf857ef87ca, entries=150, sequenceid=567, filesize=12.0 K 2024-11-19T17:14:07,454 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 689491498ff99efc87002cc59bc70ef3 in 996ms, sequenceid=567, compaction requested=true 2024-11-19T17:14:07,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:07,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:14:07,455 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:07,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:07,455 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:07,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:14:07,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:07,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:14:07,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:07,456 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40739 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:07,456 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/A is initiating minor compaction (all files) 2024-11-19T17:14:07,456 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/A in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:07,456 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/a649ac6064d749ac9c21b51760ecef49, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/3bc1afb6ab834bee8fdadcc418159728, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/de07c1b55de24175acc209dc1a1c3ff6] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=39.8 K 2024-11-19T17:14:07,456 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:07,456 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/B is initiating minor compaction (all files) 2024-11-19T17:14:07,457 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/B in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:07,457 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/05ca54b3660a4026844bd5ab415b557b, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/21e29e1c8c0e4da191c5fd07569aa9bb, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/c0f519feca0540dda4c78b51c9ad680b] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=37.4 K 2024-11-19T17:14:07,457 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting a649ac6064d749ac9c21b51760ecef49, keycount=150, bloomtype=ROW, size=13.4 K, encoding=NONE, compression=NONE, seqNum=530, earliestPutTs=1732036444462 2024-11-19T17:14:07,457 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 05ca54b3660a4026844bd5ab415b557b, keycount=150, bloomtype=ROW, size=13.4 K, encoding=NONE, compression=NONE, seqNum=530, earliestPutTs=1732036444462 2024-11-19T17:14:07,458 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bc1afb6ab834bee8fdadcc418159728, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1732036445124 2024-11-19T17:14:07,458 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 21e29e1c8c0e4da191c5fd07569aa9bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1732036445124 2024-11-19T17:14:07,458 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting c0f519feca0540dda4c78b51c9ad680b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=567, earliestPutTs=1732036446334 2024-11-19T17:14:07,459 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting de07c1b55de24175acc209dc1a1c3ff6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=567, earliestPutTs=1732036446334 2024-11-19T17:14:07,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-19T17:14:07,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:07,481 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T17:14:07,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:14:07,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:07,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:14:07,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:07,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:14:07,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:07,483 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#B#compaction#130 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:07,483 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/c439154c10694c0288d637d03e7b4328 is 50, key is test_row_0/B:col10/1732036446340/Put/seqid=0 2024-11-19T17:14:07,489 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#A#compaction#131 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:07,490 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/bd324167e8094c399dad1a66d2e9f997 is 50, key is test_row_0/A:col10/1732036446340/Put/seqid=0 2024-11-19T17:14:07,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/86c97019d73d4117bf4384a742e7b9fa is 50, key is test_row_0/A:col10/1732036446484/Put/seqid=0 2024-11-19T17:14:07,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741968_1144 (size=12301) 2024-11-19T17:14:07,520 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=582 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/86c97019d73d4117bf4384a742e7b9fa 2024-11-19T17:14:07,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741967_1143 (size=13799) 2024-11-19T17:14:07,536 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/bd324167e8094c399dad1a66d2e9f997 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/bd324167e8094c399dad1a66d2e9f997 2024-11-19T17:14:07,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741969_1145 (size=13799) 2024-11-19T17:14:07,544 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/A of 689491498ff99efc87002cc59bc70ef3 into bd324167e8094c399dad1a66d2e9f997(size=13.5 K), total size for store is 13.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:07,545 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:07,546 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/A, priority=13, startTime=1732036447454; duration=0sec 2024-11-19T17:14:07,546 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:07,546 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:A 2024-11-19T17:14:07,546 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:07,548 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:07,548 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/C is initiating minor compaction (all files) 2024-11-19T17:14:07,548 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/C in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:07,549 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/d78b2cb860d444d682e368f9afa93080, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/21ad9e57b563402a982fad1237461ad8, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/3ffe66af02894054a0debdf857ef87ca] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=37.3 K 2024-11-19T17:14:07,550 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting d78b2cb860d444d682e368f9afa93080, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=530, earliestPutTs=1732036444462 2024-11-19T17:14:07,550 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21ad9e57b563402a982fad1237461ad8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1732036445124 2024-11-19T17:14:07,550 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ffe66af02894054a0debdf857ef87ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=567, earliestPutTs=1732036446334 2024-11-19T17:14:07,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/0c38009a58a54154b78d665dee9c0b6a is 50, key is test_row_0/B:col10/1732036446484/Put/seqid=0 2024-11-19T17:14:07,560 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#C#compaction#134 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:07,561 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/afd6cb8e7d6a445081d8656add1ecaa5 is 50, key is test_row_0/C:col10/1732036446340/Put/seqid=0 2024-11-19T17:14:07,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741971_1147 (size=13731) 2024-11-19T17:14:07,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741970_1146 (size=12301) 2024-11-19T17:14:07,594 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=582 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/0c38009a58a54154b78d665dee9c0b6a 2024-11-19T17:14:07,617 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/afd6cb8e7d6a445081d8656add1ecaa5 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/afd6cb8e7d6a445081d8656add1ecaa5 2024-11-19T17:14:07,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/9042194d67674ff68a81400655588dd4 is 50, key is test_row_0/C:col10/1732036446484/Put/seqid=0 2024-11-19T17:14:07,630 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/C of 689491498ff99efc87002cc59bc70ef3 into afd6cb8e7d6a445081d8656add1ecaa5(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:07,630 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:07,631 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/C, priority=13, startTime=1732036447455; duration=0sec 2024-11-19T17:14:07,631 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:07,631 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:C 2024-11-19T17:14:07,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:14:07,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:14:07,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741972_1148 (size=12301) 2024-11-19T17:14:07,656 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=582 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/9042194d67674ff68a81400655588dd4 2024-11-19T17:14:07,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/86c97019d73d4117bf4384a742e7b9fa as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/86c97019d73d4117bf4384a742e7b9fa 2024-11-19T17:14:07,670 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/86c97019d73d4117bf4384a742e7b9fa, entries=150, sequenceid=582, filesize=12.0 K 2024-11-19T17:14:07,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/0c38009a58a54154b78d665dee9c0b6a as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/0c38009a58a54154b78d665dee9c0b6a 2024-11-19T17:14:07,680 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/0c38009a58a54154b78d665dee9c0b6a, entries=150, sequenceid=582, filesize=12.0 K 2024-11-19T17:14:07,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/9042194d67674ff68a81400655588dd4 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/9042194d67674ff68a81400655588dd4 2024-11-19T17:14:07,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-19T17:14:07,687 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/9042194d67674ff68a81400655588dd4, entries=150, sequenceid=582, filesize=12.0 K 2024-11-19T17:14:07,688 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=87.22 KB/89310 for 689491498ff99efc87002cc59bc70ef3 in 207ms, sequenceid=582, compaction requested=false 2024-11-19T17:14:07,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:07,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:07,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-11-19T17:14:07,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:14:07,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T17:14:07,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:14:07,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:07,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:14:07,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:07,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:14:07,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:07,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=30 2024-11-19T17:14:07,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/09ad4aabb7e9488c9efbacc99331d22e is 50, key is test_row_0/A:col10/1732036447687/Put/seqid=0 2024-11-19T17:14:07,697 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-19T17:14:07,697 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1400 sec 2024-11-19T17:14:07,701 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees in 1.1530 sec 2024-11-19T17:14:07,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741973_1149 (size=22065) 2024-11-19T17:14:07,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:07,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:07,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036507734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:07,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036507735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:07,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:07,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:07,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:07,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:07,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:07,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:07,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:07,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036507844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:07,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:07,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036507844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:07,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:07,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:07,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:07,952 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/c439154c10694c0288d637d03e7b4328 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/c439154c10694c0288d637d03e7b4328 2024-11-19T17:14:07,965 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/B of 689491498ff99efc87002cc59bc70ef3 into c439154c10694c0288d637d03e7b4328(size=13.5 K), total size for store is 25.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:07,965 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:07,965 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/B, priority=13, startTime=1732036447455; duration=0sec 2024-11-19T17:14:07,965 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:07,965 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:B 2024-11-19T17:14:08,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:08,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036508053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:08,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:08,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036508053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:08,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:08,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:08,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:08,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:08,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:08,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=601 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/09ad4aabb7e9488c9efbacc99331d22e 2024-11-19T17:14:08,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/294223f7b67448ca95c7f7ce74a64fcf is 50, key is test_row_0/B:col10/1732036447687/Put/seqid=0 2024-11-19T17:14:08,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741974_1150 (size=12301) 2024-11-19T17:14:08,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33842 deadline: 1732036508364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:08,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33792 deadline: 1732036508364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:08,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:08,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:08,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:08,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:08,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:08,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:08,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=601 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/294223f7b67448ca95c7f7ce74a64fcf 2024-11-19T17:14:08,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/4b1a9e14c4354d3d8a4bf704f7f3fda6 is 50, key is test_row_0/C:col10/1732036447687/Put/seqid=0 2024-11-19T17:14:08,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741975_1151 (size=12301) 2024-11-19T17:14:08,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=601 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/4b1a9e14c4354d3d8a4bf704f7f3fda6 2024-11-19T17:14:08,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/09ad4aabb7e9488c9efbacc99331d22e as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/09ad4aabb7e9488c9efbacc99331d22e 2024-11-19T17:14:08,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/09ad4aabb7e9488c9efbacc99331d22e, entries=350, sequenceid=601, filesize=21.5 K 2024-11-19T17:14:08,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/294223f7b67448ca95c7f7ce74a64fcf as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/294223f7b67448ca95c7f7ce74a64fcf 2024-11-19T17:14:08,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/294223f7b67448ca95c7f7ce74a64fcf, entries=150, sequenceid=601, filesize=12.0 K 2024-11-19T17:14:08,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/4b1a9e14c4354d3d8a4bf704f7f3fda6 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/4b1a9e14c4354d3d8a4bf704f7f3fda6 2024-11-19T17:14:08,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/4b1a9e14c4354d3d8a4bf704f7f3fda6, entries=150, sequenceid=601, filesize=12.0 K 2024-11-19T17:14:08,597 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 689491498ff99efc87002cc59bc70ef3 in 908ms, sequenceid=601, compaction requested=true 2024-11-19T17:14:08,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:08,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:14:08,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:08,599 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:08,599 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:08,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:14:08,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:08,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 689491498ff99efc87002cc59bc70ef3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:14:08,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:08,600 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:08,600 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/B is initiating minor compaction (all files) 2024-11-19T17:14:08,600 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/B in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:08,601 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/c439154c10694c0288d637d03e7b4328, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/0c38009a58a54154b78d665dee9c0b6a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/294223f7b67448ca95c7f7ce74a64fcf] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=37.5 K 2024-11-19T17:14:08,601 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 48165 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:08,601 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/A is initiating minor compaction (all files) 2024-11-19T17:14:08,601 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/A in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:08,601 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting c439154c10694c0288d637d03e7b4328, keycount=150, bloomtype=ROW, size=13.5 K, encoding=NONE, compression=NONE, seqNum=567, earliestPutTs=1732036446334 2024-11-19T17:14:08,601 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/bd324167e8094c399dad1a66d2e9f997, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/86c97019d73d4117bf4384a742e7b9fa, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/09ad4aabb7e9488c9efbacc99331d22e] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=47.0 K 2024-11-19T17:14:08,604 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c38009a58a54154b78d665dee9c0b6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=582, earliestPutTs=1732036446484 2024-11-19T17:14:08,604 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd324167e8094c399dad1a66d2e9f997, keycount=150, bloomtype=ROW, size=13.5 K, encoding=NONE, compression=NONE, seqNum=567, earliestPutTs=1732036446334 2024-11-19T17:14:08,605 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 294223f7b67448ca95c7f7ce74a64fcf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=601, earliestPutTs=1732036447677 2024-11-19T17:14:08,606 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86c97019d73d4117bf4384a742e7b9fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=582, earliestPutTs=1732036446484 2024-11-19T17:14:08,607 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09ad4aabb7e9488c9efbacc99331d22e, keycount=350, bloomtype=ROW, size=21.5 K, encoding=NONE, compression=NONE, seqNum=601, earliestPutTs=1732036447668 2024-11-19T17:14:08,623 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#B#compaction#139 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:08,623 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/83d5daadbc504e75b43f263a1451635d is 50, key is test_row_0/B:col10/1732036447687/Put/seqid=0 2024-11-19T17:14:08,626 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#A#compaction#140 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:08,627 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/26130a9d43784ca48ea3db16542fc207 is 50, key is test_row_0/A:col10/1732036447687/Put/seqid=0 2024-11-19T17:14:08,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741976_1152 (size=13901) 2024-11-19T17:14:08,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741977_1153 (size=13901) 2024-11-19T17:14:08,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-19T17:14:08,694 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:14:08,695 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:14:08,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=31, table=TestAcidGuarantees 2024-11-19T17:14:08,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-19T17:14:08,698 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=31, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=31, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:14:08,699 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=31, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=31, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:14:08,699 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:14:08,761 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-19T17:14:08,761 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:14:08,764 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-19T17:14:08,764 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:14:08,766 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-19T17:14:08,766 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:14:08,768 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:08,768 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:08,768 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:08,769 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:08,769 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:08,769 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:14:08,769 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:14:08,769 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:08,769 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:14:08,770 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-19T17:14:08,770 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:14:08,771 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:08,771 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:08,771 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:14:08,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-19T17:14:08,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-19T17:14:08,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:08,854 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T17:14:08,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:14:08,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:08,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:14:08,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:08,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:14:08,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:08,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/6a7ad2fc5e2d469d96e62dd60ecdbc59 is 50, key is test_row_0/A:col10/1732036447723/Put/seqid=0 2024-11-19T17:14:08,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741978_1154 (size=12301) 2024-11-19T17:14:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:14:08,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. as already flushing 2024-11-19T17:14:08,876 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-19T17:14:08,876 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-19T17:14:08,876 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:14:08,876 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:08,876 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:14:08,876 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:08,876 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:08,876 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:14:08,876 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:08,876 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:14:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-19T17:14:09,055 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/83d5daadbc504e75b43f263a1451635d as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/83d5daadbc504e75b43f263a1451635d 2024-11-19T17:14:09,060 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/B of 689491498ff99efc87002cc59bc70ef3 into 83d5daadbc504e75b43f263a1451635d(size=13.6 K), total size for store is 13.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:09,060 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:09,060 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/B, priority=13, startTime=1732036448599; duration=0sec 2024-11-19T17:14:09,060 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:09,060 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:B 2024-11-19T17:14:09,060 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:09,061 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:09,061 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 689491498ff99efc87002cc59bc70ef3/C is initiating minor compaction (all files) 2024-11-19T17:14:09,061 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 689491498ff99efc87002cc59bc70ef3/C in TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:09,062 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/afd6cb8e7d6a445081d8656add1ecaa5, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/9042194d67674ff68a81400655588dd4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/4b1a9e14c4354d3d8a4bf704f7f3fda6] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp, totalSize=37.4 K 2024-11-19T17:14:09,062 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting afd6cb8e7d6a445081d8656add1ecaa5, keycount=150, bloomtype=ROW, size=13.4 K, encoding=NONE, compression=NONE, seqNum=567, earliestPutTs=1732036446334 2024-11-19T17:14:09,062 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 9042194d67674ff68a81400655588dd4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=582, earliestPutTs=1732036446484 2024-11-19T17:14:09,063 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b1a9e14c4354d3d8a4bf704f7f3fda6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=601, earliestPutTs=1732036447677 2024-11-19T17:14:09,068 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/26130a9d43784ca48ea3db16542fc207 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/26130a9d43784ca48ea3db16542fc207 2024-11-19T17:14:09,069 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 689491498ff99efc87002cc59bc70ef3#C#compaction#142 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:09,070 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/eec3cee9a9f948e6ac9315b51911c3d4 is 50, key is test_row_0/C:col10/1732036447687/Put/seqid=0 2024-11-19T17:14:09,073 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/A of 689491498ff99efc87002cc59bc70ef3 into 26130a9d43784ca48ea3db16542fc207(size=13.6 K), total size for store is 13.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:09,073 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:09,073 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/A, priority=13, startTime=1732036448599; duration=0sec 2024-11-19T17:14:09,073 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:09,073 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:A 2024-11-19T17:14:09,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741979_1155 (size=13833) 2024-11-19T17:14:09,264 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/6a7ad2fc5e2d469d96e62dd60ecdbc59 2024-11-19T17:14:09,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/b092411092d142beb774f60a19e937c5 is 50, key is test_row_0/B:col10/1732036447723/Put/seqid=0 2024-11-19T17:14:09,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741980_1156 (size=12301) 2024-11-19T17:14:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-19T17:14:09,475 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-19T17:14:09,476 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:14:09,476 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:09,476 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:09,476 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:14:09,479 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/eec3cee9a9f948e6ac9315b51911c3d4 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/eec3cee9a9f948e6ac9315b51911c3d4 2024-11-19T17:14:09,484 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 689491498ff99efc87002cc59bc70ef3/C of 689491498ff99efc87002cc59bc70ef3 into eec3cee9a9f948e6ac9315b51911c3d4(size=13.5 K), total size for store is 13.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:09,484 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:09,484 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., storeName=689491498ff99efc87002cc59bc70ef3/C, priority=13, startTime=1732036448599; duration=0sec 2024-11-19T17:14:09,484 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:09,484 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 689491498ff99efc87002cc59bc70ef3:C 2024-11-19T17:14:09,515 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-19T17:14:09,515 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:14:09,515 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:09,515 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:09,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:14:09,525 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-19T17:14:09,525 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:14:09,525 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:09,525 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:09,525 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:14:09,676 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/b092411092d142beb774f60a19e937c5 2024-11-19T17:14:09,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/8b42c435f3564e50a015fb00c388871f is 50, key is test_row_0/C:col10/1732036447723/Put/seqid=0 2024-11-19T17:14:09,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741981_1157 (size=12301) 2024-11-19T17:14:09,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-19T17:14:10,088 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/8b42c435f3564e50a015fb00c388871f 2024-11-19T17:14:10,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/6a7ad2fc5e2d469d96e62dd60ecdbc59 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/6a7ad2fc5e2d469d96e62dd60ecdbc59 2024-11-19T17:14:10,097 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/6a7ad2fc5e2d469d96e62dd60ecdbc59, entries=150, sequenceid=618, filesize=12.0 K 2024-11-19T17:14:10,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/b092411092d142beb774f60a19e937c5 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/b092411092d142beb774f60a19e937c5 2024-11-19T17:14:10,101 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/b092411092d142beb774f60a19e937c5, entries=150, sequenceid=618, filesize=12.0 K 2024-11-19T17:14:10,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/8b42c435f3564e50a015fb00c388871f as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/8b42c435f3564e50a015fb00c388871f 2024-11-19T17:14:10,106 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/8b42c435f3564e50a015fb00c388871f, entries=150, sequenceid=618, filesize=12.0 K 2024-11-19T17:14:10,106 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=33.54 KB/34350 for 689491498ff99efc87002cc59bc70ef3 in 1252ms, sequenceid=618, compaction requested=false 2024-11-19T17:14:10,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2603): Flush status journal for 689491498ff99efc87002cc59bc70ef3: 2024-11-19T17:14:10,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:10,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=32 2024-11-19T17:14:10,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=32 2024-11-19T17:14:10,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-19T17:14:10,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4090 sec 2024-11-19T17:14:10,111 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=31, table=TestAcidGuarantees in 1.4150 sec 2024-11-19T17:14:10,708 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T17:14:10,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-19T17:14:10,844 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:14:10,844 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T17:14:10,844 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-19T17:14:10,844 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 113 2024-11-19T17:14:10,844 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-11-19T17:14:10,844 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-19T17:14:10,844 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-19T17:14:10,844 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T17:14:10,844 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2640 2024-11-19T17:14:10,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2515 2024-11-19T17:14:10,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T17:14:10,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1179 2024-11-19T17:14:10,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3533 rows 2024-11-19T17:14:10,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1181 2024-11-19T17:14:10,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3541 rows 2024-11-19T17:14:10,845 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T17:14:10,845 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T17:14:10,845 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:10,845 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:10,845 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:14:10,849 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T17:14:10,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T17:14:10,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-19T17:14:10,857 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732036450857"}]},"ts":"1732036450857"} 2024-11-19T17:14:10,859 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T17:14:10,859 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T17:14:10,861 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T17:14:10,865 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=689491498ff99efc87002cc59bc70ef3, UNASSIGN}] 2024-11-19T17:14:10,866 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=689491498ff99efc87002cc59bc70ef3, UNASSIGN 2024-11-19T17:14:10,867 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=689491498ff99efc87002cc59bc70ef3, regionState=CLOSING, regionLocation=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:10,869 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=689491498ff99efc87002cc59bc70ef3, UNASSIGN because future has completed 2024-11-19T17:14:10,870 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T17:14:10,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473}] 2024-11-19T17:14:10,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-19T17:14:11,027 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:14:11,027 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T17:14:11,027 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 689491498ff99efc87002cc59bc70ef3, disabling compactions & flushes 2024-11-19T17:14:11,028 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:11,028 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:11,028 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. after waiting 0 ms 2024-11-19T17:14:11,028 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:11,028 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(2902): Flushing 689491498ff99efc87002cc59bc70ef3 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T17:14:11,028 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=A 2024-11-19T17:14:11,028 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:11,028 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=B 2024-11-19T17:14:11,028 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:11,028 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 689491498ff99efc87002cc59bc70ef3, store=C 2024-11-19T17:14:11,028 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:11,032 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/2337f4eb623140fcb40258ae7dfb843d is 50, key is test_row_1/A:col10/1732036449524/Put/seqid=0 2024-11-19T17:14:11,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741982_1158 (size=9857) 2024-11-19T17:14:11,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-19T17:14:11,437 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=629 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/2337f4eb623140fcb40258ae7dfb843d 2024-11-19T17:14:11,445 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/856be8a343f94645bd05f5b9e0105b65 is 50, key is test_row_1/B:col10/1732036449524/Put/seqid=0 2024-11-19T17:14:11,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741983_1159 (size=9857) 2024-11-19T17:14:11,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-19T17:14:11,849 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=629 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/856be8a343f94645bd05f5b9e0105b65 2024-11-19T17:14:11,857 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/bf3ab40cb40e4c2bb8d7f123acaceb49 is 50, key is test_row_1/C:col10/1732036449524/Put/seqid=0 2024-11-19T17:14:11,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741984_1160 (size=9857) 2024-11-19T17:14:11,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-19T17:14:12,261 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=629 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/bf3ab40cb40e4c2bb8d7f123acaceb49 2024-11-19T17:14:12,266 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/A/2337f4eb623140fcb40258ae7dfb843d as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/2337f4eb623140fcb40258ae7dfb843d 2024-11-19T17:14:12,270 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/2337f4eb623140fcb40258ae7dfb843d, entries=100, sequenceid=629, filesize=9.6 K 2024-11-19T17:14:12,271 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/B/856be8a343f94645bd05f5b9e0105b65 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/856be8a343f94645bd05f5b9e0105b65 2024-11-19T17:14:12,275 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/856be8a343f94645bd05f5b9e0105b65, entries=100, sequenceid=629, filesize=9.6 K 2024-11-19T17:14:12,276 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/.tmp/C/bf3ab40cb40e4c2bb8d7f123acaceb49 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/bf3ab40cb40e4c2bb8d7f123acaceb49 2024-11-19T17:14:12,279 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/bf3ab40cb40e4c2bb8d7f123acaceb49, entries=100, sequenceid=629, filesize=9.6 K 2024-11-19T17:14:12,280 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 689491498ff99efc87002cc59bc70ef3 in 1252ms, sequenceid=629, compaction requested=true 2024-11-19T17:14:12,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/412a1bba31264b349a9732ae7b8883d4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/0ada863ef623421fb57eeda223ef7e7e, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/2be3c46b72b643fbab1edc2e6dcfc906, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/85712a4f27bf4cff916dbd01fbb2f3e4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/7249596e0ace49678916e53cfb1967ae, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/fcac6f6e06b94bd6ae0958db743a9cec, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/f967a0c4215244cd93bf40c72d558b9c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/e7b7b24a474145f294896866a61ed477, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/288ea15fb8d143979acb82930193f480, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/124027852d094f65b31f1a7f1366236a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/a85b36766c0945c390d9ee817ab51d4e, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/72806d71bc8e48c994d86445874b540f, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/c42694a1dd5d48b4abc5c1a2fe6deeb2, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/2dabb56049354be38dd1b7845157abfd, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/a28a4e42c44d4f998239fc14d7501016, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/06db09cc1e7144a7b7649ad2d2997d2e, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/ffd3d2cacbce4bacaa833b8f875e8cab, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/497374115a8b4a6d872ca3a2b2dcc1a8, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/bcf4efd778ce4c858f601e910e479989, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/d745487350e547b6a78ed62d436008a4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/6e691ee35a464b21a38c67f73b892968, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/5d2254c632d74e83941979969445f7a8, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/e0c634da53794865b25c6ce2cffec44a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/bea616e0004b4ef3adc4cf36ff6dd8a8, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/2f7baaa8744c493f97be3d75aa54e151, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/afa60b4eae5b4849b42b83d32b47f42b, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/e461b5c52c9049e1bdc3686b3f129469, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/db845d7b4f354ceab7b3fbac65437a91, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/c1952dcf708f4328a4714bc06aefcd3d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/2dfc74eac7f44c918d43b084b7cc9db5, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/be36aab5a89b48c1ac0b4c535e877f5a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/feb0f0467cb74cee93352825c25e9bde, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/02ffecb125f34fe0a938a11b9a9eb751, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/6c3445ecc48947b7ba1f57518f2a85c4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/7751205b907c44648258e768d54d3a50, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/c97b632038b848c0a6014cf8c8a379c3, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/3d730d95ce6f4a30b037621edde132f3, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/1fb27786b9354178929d60598bdcf52f, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/ad484189020047bc9b50a6caed1555bf, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/b94d53c1e17644cbbdbbd371d0cd5797, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/f04b573608f74230a4fece29df39178c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/a649ac6064d749ac9c21b51760ecef49, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/3bc1afb6ab834bee8fdadcc418159728, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/de07c1b55de24175acc209dc1a1c3ff6, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/bd324167e8094c399dad1a66d2e9f997, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/86c97019d73d4117bf4384a742e7b9fa, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/09ad4aabb7e9488c9efbacc99331d22e] to archive 2024-11-19T17:14:12,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:14:12,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/412a1bba31264b349a9732ae7b8883d4 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/412a1bba31264b349a9732ae7b8883d4 2024-11-19T17:14:12,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/0ada863ef623421fb57eeda223ef7e7e to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/0ada863ef623421fb57eeda223ef7e7e 2024-11-19T17:14:12,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/2be3c46b72b643fbab1edc2e6dcfc906 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/2be3c46b72b643fbab1edc2e6dcfc906 2024-11-19T17:14:12,293 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/85712a4f27bf4cff916dbd01fbb2f3e4 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/85712a4f27bf4cff916dbd01fbb2f3e4 2024-11-19T17:14:12,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/7249596e0ace49678916e53cfb1967ae to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/7249596e0ace49678916e53cfb1967ae 2024-11-19T17:14:12,296 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/fcac6f6e06b94bd6ae0958db743a9cec to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/fcac6f6e06b94bd6ae0958db743a9cec 2024-11-19T17:14:12,297 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/f967a0c4215244cd93bf40c72d558b9c to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/f967a0c4215244cd93bf40c72d558b9c 2024-11-19T17:14:12,298 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/e7b7b24a474145f294896866a61ed477 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/e7b7b24a474145f294896866a61ed477 2024-11-19T17:14:12,299 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/288ea15fb8d143979acb82930193f480 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/288ea15fb8d143979acb82930193f480 2024-11-19T17:14:12,300 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/124027852d094f65b31f1a7f1366236a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/124027852d094f65b31f1a7f1366236a 2024-11-19T17:14:12,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/a85b36766c0945c390d9ee817ab51d4e to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/a85b36766c0945c390d9ee817ab51d4e 2024-11-19T17:14:12,302 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/72806d71bc8e48c994d86445874b540f to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/72806d71bc8e48c994d86445874b540f 2024-11-19T17:14:12,303 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/c42694a1dd5d48b4abc5c1a2fe6deeb2 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/c42694a1dd5d48b4abc5c1a2fe6deeb2 2024-11-19T17:14:12,305 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/2dabb56049354be38dd1b7845157abfd to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/2dabb56049354be38dd1b7845157abfd 2024-11-19T17:14:12,306 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/a28a4e42c44d4f998239fc14d7501016 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/a28a4e42c44d4f998239fc14d7501016 2024-11-19T17:14:12,307 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/06db09cc1e7144a7b7649ad2d2997d2e to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/06db09cc1e7144a7b7649ad2d2997d2e 2024-11-19T17:14:12,308 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/ffd3d2cacbce4bacaa833b8f875e8cab to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/ffd3d2cacbce4bacaa833b8f875e8cab 2024-11-19T17:14:12,309 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/497374115a8b4a6d872ca3a2b2dcc1a8 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/497374115a8b4a6d872ca3a2b2dcc1a8 2024-11-19T17:14:12,311 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/bcf4efd778ce4c858f601e910e479989 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/bcf4efd778ce4c858f601e910e479989 2024-11-19T17:14:12,312 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/d745487350e547b6a78ed62d436008a4 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/d745487350e547b6a78ed62d436008a4 2024-11-19T17:14:12,313 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/6e691ee35a464b21a38c67f73b892968 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/6e691ee35a464b21a38c67f73b892968 2024-11-19T17:14:12,314 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/5d2254c632d74e83941979969445f7a8 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/5d2254c632d74e83941979969445f7a8 2024-11-19T17:14:12,316 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/e0c634da53794865b25c6ce2cffec44a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/e0c634da53794865b25c6ce2cffec44a 2024-11-19T17:14:12,317 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/bea616e0004b4ef3adc4cf36ff6dd8a8 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/bea616e0004b4ef3adc4cf36ff6dd8a8 2024-11-19T17:14:12,318 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/2f7baaa8744c493f97be3d75aa54e151 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/2f7baaa8744c493f97be3d75aa54e151 2024-11-19T17:14:12,319 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/afa60b4eae5b4849b42b83d32b47f42b to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/afa60b4eae5b4849b42b83d32b47f42b 2024-11-19T17:14:12,320 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/e461b5c52c9049e1bdc3686b3f129469 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/e461b5c52c9049e1bdc3686b3f129469 2024-11-19T17:14:12,321 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/db845d7b4f354ceab7b3fbac65437a91 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/db845d7b4f354ceab7b3fbac65437a91 2024-11-19T17:14:12,322 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/c1952dcf708f4328a4714bc06aefcd3d to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/c1952dcf708f4328a4714bc06aefcd3d 2024-11-19T17:14:12,324 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/2dfc74eac7f44c918d43b084b7cc9db5 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/2dfc74eac7f44c918d43b084b7cc9db5 2024-11-19T17:14:12,325 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/be36aab5a89b48c1ac0b4c535e877f5a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/be36aab5a89b48c1ac0b4c535e877f5a 2024-11-19T17:14:12,326 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/feb0f0467cb74cee93352825c25e9bde to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/feb0f0467cb74cee93352825c25e9bde 2024-11-19T17:14:12,327 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/02ffecb125f34fe0a938a11b9a9eb751 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/02ffecb125f34fe0a938a11b9a9eb751 2024-11-19T17:14:12,328 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/6c3445ecc48947b7ba1f57518f2a85c4 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/6c3445ecc48947b7ba1f57518f2a85c4 2024-11-19T17:14:12,330 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/7751205b907c44648258e768d54d3a50 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/7751205b907c44648258e768d54d3a50 2024-11-19T17:14:12,331 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/c97b632038b848c0a6014cf8c8a379c3 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/c97b632038b848c0a6014cf8c8a379c3 2024-11-19T17:14:12,332 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/3d730d95ce6f4a30b037621edde132f3 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/3d730d95ce6f4a30b037621edde132f3 2024-11-19T17:14:12,333 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/1fb27786b9354178929d60598bdcf52f to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/1fb27786b9354178929d60598bdcf52f 2024-11-19T17:14:12,334 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/ad484189020047bc9b50a6caed1555bf to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/ad484189020047bc9b50a6caed1555bf 2024-11-19T17:14:12,336 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/b94d53c1e17644cbbdbbd371d0cd5797 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/b94d53c1e17644cbbdbbd371d0cd5797 2024-11-19T17:14:12,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/f04b573608f74230a4fece29df39178c to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/f04b573608f74230a4fece29df39178c 2024-11-19T17:14:12,338 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/a649ac6064d749ac9c21b51760ecef49 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/a649ac6064d749ac9c21b51760ecef49 2024-11-19T17:14:12,339 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/3bc1afb6ab834bee8fdadcc418159728 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/3bc1afb6ab834bee8fdadcc418159728 2024-11-19T17:14:12,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/de07c1b55de24175acc209dc1a1c3ff6 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/de07c1b55de24175acc209dc1a1c3ff6 2024-11-19T17:14:12,342 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/bd324167e8094c399dad1a66d2e9f997 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/bd324167e8094c399dad1a66d2e9f997 2024-11-19T17:14:12,343 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/86c97019d73d4117bf4384a742e7b9fa to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/86c97019d73d4117bf4384a742e7b9fa 2024-11-19T17:14:12,344 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/09ad4aabb7e9488c9efbacc99331d22e to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/09ad4aabb7e9488c9efbacc99331d22e 2024-11-19T17:14:12,359 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/6d29c6a1872d4fbf99fb5b889c0af1a7, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/4293bbf4b68f4fc895a6cfec00943ed7, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/81f05dcf80994b19940050dd4fd2a741, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/eb0a4526815a4ab3813ad56038f36352, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/8d7e31c4f3ec4f999a592e15f75900dc, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/4dd0ecf1fadd436f9ef2080a93fa34af, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/eee8fda4e1804de3a28dc947254b49d6, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/f4f7b01b13594de196666d254479b8c7, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/b5cbc35d021a4d82a51b7148d4f6feb4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/31cd7e26846f4b2480e5697e9ae629ef, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/5006982c214c4fdcb9a8573d516b3455, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/223e780d2b5d4d0797a279e74e87eb01, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/8f500f1c53304da08352ef8421324b2a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/a27909cc86144dd8a67d2aa3ff9a9ff9, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/56a4aae1a0cd43c482e2f719e3738040, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/71a38e6db75f4db68b263df2e9eb85ab, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/81049fa87db540359ca84976179d05d6, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/b21d7f1fb8644180806a3f5a38742ea5, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/874535960c08428c96d2fac89d7eb6f2, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/8715444661c6438e8f3f849cabe024ac, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/34b33fdb706e40e395473f116af943c9, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/4935923d8b4647f4b9136be177b6f60c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/54fb7313e9074b78aa72add90ca0dc8d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/ec8874209090488dbe1e8371e7bda713, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/6bb0b9acc6a843deb76a30799dfb8f45, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/105f79f2123f4f99952b16acf9e92fab, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/80c22b9b22ad408cafd19ce39f5080be, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/778389b0eee74594b8f0c0f2fd52ce3f, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/98186ecaa2234dfcb8128ab844de1455, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/a70908e523984b6ca423bae51f9b3064, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/faa0c1ef838242d69cad78cdedf215f8, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/59a1baac523e46deb117d25bab3da0c5, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/21b9a06be294438582c16a8e671fb8f4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/0187133a8e004438a281a70320cdcb12, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/02f94b05301c4606ae362c17470dcf79, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/98937ed889014f0a96e09886b39dfb50, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/96dcf8d0294f4a1d8255fa3551ee5627, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/cdba7d8e50b74ed5abd8833dc16bef15, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/68762490557546de99d2ae4ab37398e0, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/d670c5cf48ca4308bcf95266b5a968b3, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/05ca54b3660a4026844bd5ab415b557b, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/147abe740d9e42fa8fa8dfcabe93b159, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/21e29e1c8c0e4da191c5fd07569aa9bb, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/c439154c10694c0288d637d03e7b4328, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/c0f519feca0540dda4c78b51c9ad680b, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/0c38009a58a54154b78d665dee9c0b6a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/294223f7b67448ca95c7f7ce74a64fcf] to archive 2024-11-19T17:14:12,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:14:12,362 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/6d29c6a1872d4fbf99fb5b889c0af1a7 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/6d29c6a1872d4fbf99fb5b889c0af1a7 2024-11-19T17:14:12,364 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/4293bbf4b68f4fc895a6cfec00943ed7 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/4293bbf4b68f4fc895a6cfec00943ed7 2024-11-19T17:14:12,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/81f05dcf80994b19940050dd4fd2a741 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/81f05dcf80994b19940050dd4fd2a741 2024-11-19T17:14:12,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/eb0a4526815a4ab3813ad56038f36352 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/eb0a4526815a4ab3813ad56038f36352 2024-11-19T17:14:12,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/8d7e31c4f3ec4f999a592e15f75900dc to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/8d7e31c4f3ec4f999a592e15f75900dc 2024-11-19T17:14:12,368 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/4dd0ecf1fadd436f9ef2080a93fa34af to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/4dd0ecf1fadd436f9ef2080a93fa34af 2024-11-19T17:14:12,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/eee8fda4e1804de3a28dc947254b49d6 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/eee8fda4e1804de3a28dc947254b49d6 2024-11-19T17:14:12,370 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/f4f7b01b13594de196666d254479b8c7 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/f4f7b01b13594de196666d254479b8c7 2024-11-19T17:14:12,371 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/b5cbc35d021a4d82a51b7148d4f6feb4 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/b5cbc35d021a4d82a51b7148d4f6feb4 2024-11-19T17:14:12,372 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/31cd7e26846f4b2480e5697e9ae629ef to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/31cd7e26846f4b2480e5697e9ae629ef 2024-11-19T17:14:12,373 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/5006982c214c4fdcb9a8573d516b3455 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/5006982c214c4fdcb9a8573d516b3455 2024-11-19T17:14:12,374 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/223e780d2b5d4d0797a279e74e87eb01 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/223e780d2b5d4d0797a279e74e87eb01 2024-11-19T17:14:12,375 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/8f500f1c53304da08352ef8421324b2a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/8f500f1c53304da08352ef8421324b2a 2024-11-19T17:14:12,377 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/a27909cc86144dd8a67d2aa3ff9a9ff9 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/a27909cc86144dd8a67d2aa3ff9a9ff9 2024-11-19T17:14:12,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/56a4aae1a0cd43c482e2f719e3738040 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/56a4aae1a0cd43c482e2f719e3738040 2024-11-19T17:14:12,379 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/71a38e6db75f4db68b263df2e9eb85ab to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/71a38e6db75f4db68b263df2e9eb85ab 2024-11-19T17:14:12,380 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/81049fa87db540359ca84976179d05d6 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/81049fa87db540359ca84976179d05d6 2024-11-19T17:14:12,381 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/b21d7f1fb8644180806a3f5a38742ea5 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/b21d7f1fb8644180806a3f5a38742ea5 2024-11-19T17:14:12,382 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/874535960c08428c96d2fac89d7eb6f2 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/874535960c08428c96d2fac89d7eb6f2 2024-11-19T17:14:12,383 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/8715444661c6438e8f3f849cabe024ac to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/8715444661c6438e8f3f849cabe024ac 2024-11-19T17:14:12,384 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/34b33fdb706e40e395473f116af943c9 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/34b33fdb706e40e395473f116af943c9 2024-11-19T17:14:12,385 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/4935923d8b4647f4b9136be177b6f60c to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/4935923d8b4647f4b9136be177b6f60c 2024-11-19T17:14:12,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/54fb7313e9074b78aa72add90ca0dc8d to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/54fb7313e9074b78aa72add90ca0dc8d 2024-11-19T17:14:12,387 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/ec8874209090488dbe1e8371e7bda713 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/ec8874209090488dbe1e8371e7bda713 2024-11-19T17:14:12,388 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/6bb0b9acc6a843deb76a30799dfb8f45 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/6bb0b9acc6a843deb76a30799dfb8f45 2024-11-19T17:14:12,389 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/105f79f2123f4f99952b16acf9e92fab to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/105f79f2123f4f99952b16acf9e92fab 2024-11-19T17:14:12,390 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/80c22b9b22ad408cafd19ce39f5080be to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/80c22b9b22ad408cafd19ce39f5080be 2024-11-19T17:14:12,391 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/778389b0eee74594b8f0c0f2fd52ce3f to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/778389b0eee74594b8f0c0f2fd52ce3f 2024-11-19T17:14:12,393 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/98186ecaa2234dfcb8128ab844de1455 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/98186ecaa2234dfcb8128ab844de1455 2024-11-19T17:14:12,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/a70908e523984b6ca423bae51f9b3064 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/a70908e523984b6ca423bae51f9b3064 2024-11-19T17:14:12,395 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/faa0c1ef838242d69cad78cdedf215f8 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/faa0c1ef838242d69cad78cdedf215f8 2024-11-19T17:14:12,396 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/59a1baac523e46deb117d25bab3da0c5 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/59a1baac523e46deb117d25bab3da0c5 2024-11-19T17:14:12,397 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/21b9a06be294438582c16a8e671fb8f4 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/21b9a06be294438582c16a8e671fb8f4 2024-11-19T17:14:12,398 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/0187133a8e004438a281a70320cdcb12 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/0187133a8e004438a281a70320cdcb12 2024-11-19T17:14:12,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/02f94b05301c4606ae362c17470dcf79 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/02f94b05301c4606ae362c17470dcf79 2024-11-19T17:14:12,400 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/98937ed889014f0a96e09886b39dfb50 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/98937ed889014f0a96e09886b39dfb50 2024-11-19T17:14:12,401 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/96dcf8d0294f4a1d8255fa3551ee5627 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/96dcf8d0294f4a1d8255fa3551ee5627 2024-11-19T17:14:12,402 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/cdba7d8e50b74ed5abd8833dc16bef15 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/cdba7d8e50b74ed5abd8833dc16bef15 2024-11-19T17:14:12,403 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/68762490557546de99d2ae4ab37398e0 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/68762490557546de99d2ae4ab37398e0 2024-11-19T17:14:12,404 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/d670c5cf48ca4308bcf95266b5a968b3 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/d670c5cf48ca4308bcf95266b5a968b3 2024-11-19T17:14:12,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/05ca54b3660a4026844bd5ab415b557b to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/05ca54b3660a4026844bd5ab415b557b 2024-11-19T17:14:12,407 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/147abe740d9e42fa8fa8dfcabe93b159 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/147abe740d9e42fa8fa8dfcabe93b159 2024-11-19T17:14:12,408 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/21e29e1c8c0e4da191c5fd07569aa9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/21e29e1c8c0e4da191c5fd07569aa9bb 2024-11-19T17:14:12,409 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/c439154c10694c0288d637d03e7b4328 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/c439154c10694c0288d637d03e7b4328 2024-11-19T17:14:12,410 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/c0f519feca0540dda4c78b51c9ad680b to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/c0f519feca0540dda4c78b51c9ad680b 2024-11-19T17:14:12,411 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/0c38009a58a54154b78d665dee9c0b6a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/0c38009a58a54154b78d665dee9c0b6a 2024-11-19T17:14:12,412 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/294223f7b67448ca95c7f7ce74a64fcf to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/294223f7b67448ca95c7f7ce74a64fcf 2024-11-19T17:14:12,414 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/cc8da696192f4cbab47a81a1dc4b18ae, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/a89c4701d5334e999ae4acc0fe01c599, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/5dc03f97a5cf45659bf93b6865e66d4b, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/92d6f7532dc446d1b0706a077a2944c6, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/166069766cc34275b4f48a3e36533d30, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/e100b01ecbb04678b859edad0035a8de, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/f8fa352c0ca64395bbc955a8915ca0a2, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/4b371b28decc49909b316e97f58462ac, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/3da0a5910587412f8c2ac89ea54e53b6, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/6a84aca6083342f88410477c5a9996a5, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/721c9cbc2ce24532b4d82b771892c98b, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/c39cbaa5cf4643819d474a2766b24087, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/55da356648e94aabb9e98d5eb68d5fab, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/75068864d62f4d779e6a8c5163ae0e8c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/9ac9b1fb30bc40a2bb7bdc5b62b5a8ff, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/9f2217e83a6b4ba1a876ea683ac9df49, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/489e641fd1e8449c94379a69f7c81fd8, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/6c7ac8a6cb614729bef8c602d6eb5275, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/edcb6f33a4a3465599e144a1c7fad27a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/aa7d720c06e44a8d87af784b9ff7064f, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/c2349cf18af241ab8559cbbfd4192a63, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/2d216efd1aa4404e87ee0ea4d56d6923, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/2697b17a65034698b21227949bc17a7c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/c01ab535b5e649608ed82fa436f5b38c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/34d9471bddd84730ac1f342ef5a219ed, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/7935e682368a45e39fab7c52c1ba5a6c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/03e2261148df447db2e791124dce802c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/4d070322eac742eb83a3ecd3a8423c6a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/d3b79abd5ab94cf7a475cf741342d458, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/c0bb45ca494c414fa1ad3f6dcda78422, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/a8f6a14d1996447d88c8ee5fd930ebda, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/a31a066021b34294b1929d67671cf008, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/3946624d13164aff9de7abd136999c7f, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/fdb799d488884069931f3e68489f59ce, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/89f67b4d082d4c5ba5c0163bf8cb12ce, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/464cb1d766404842b15ea4c183019a0f, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/599d715747f140cea68cc837063930bd, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/b8783eeb558b4c7e930fa136023f9fa2, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/d78b2cb860d444d682e368f9afa93080, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/de22ffa6d3444d7e9bc11a68465d7b94, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/21ad9e57b563402a982fad1237461ad8, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/afd6cb8e7d6a445081d8656add1ecaa5, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/3ffe66af02894054a0debdf857ef87ca, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/9042194d67674ff68a81400655588dd4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/4b1a9e14c4354d3d8a4bf704f7f3fda6] to archive 2024-11-19T17:14:12,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:14:12,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/cc8da696192f4cbab47a81a1dc4b18ae to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/cc8da696192f4cbab47a81a1dc4b18ae 2024-11-19T17:14:12,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/a89c4701d5334e999ae4acc0fe01c599 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/a89c4701d5334e999ae4acc0fe01c599 2024-11-19T17:14:12,419 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/5dc03f97a5cf45659bf93b6865e66d4b to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/5dc03f97a5cf45659bf93b6865e66d4b 2024-11-19T17:14:12,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/92d6f7532dc446d1b0706a077a2944c6 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/92d6f7532dc446d1b0706a077a2944c6 2024-11-19T17:14:12,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/166069766cc34275b4f48a3e36533d30 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/166069766cc34275b4f48a3e36533d30 2024-11-19T17:14:12,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/e100b01ecbb04678b859edad0035a8de to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/e100b01ecbb04678b859edad0035a8de 2024-11-19T17:14:12,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/f8fa352c0ca64395bbc955a8915ca0a2 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/f8fa352c0ca64395bbc955a8915ca0a2 2024-11-19T17:14:12,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/4b371b28decc49909b316e97f58462ac to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/4b371b28decc49909b316e97f58462ac 2024-11-19T17:14:12,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/3da0a5910587412f8c2ac89ea54e53b6 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/3da0a5910587412f8c2ac89ea54e53b6 2024-11-19T17:14:12,427 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/6a84aca6083342f88410477c5a9996a5 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/6a84aca6083342f88410477c5a9996a5 2024-11-19T17:14:12,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/721c9cbc2ce24532b4d82b771892c98b to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/721c9cbc2ce24532b4d82b771892c98b 2024-11-19T17:14:12,429 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/c39cbaa5cf4643819d474a2766b24087 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/c39cbaa5cf4643819d474a2766b24087 2024-11-19T17:14:12,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/55da356648e94aabb9e98d5eb68d5fab to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/55da356648e94aabb9e98d5eb68d5fab 2024-11-19T17:14:12,432 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/75068864d62f4d779e6a8c5163ae0e8c to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/75068864d62f4d779e6a8c5163ae0e8c 2024-11-19T17:14:12,433 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/9ac9b1fb30bc40a2bb7bdc5b62b5a8ff to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/9ac9b1fb30bc40a2bb7bdc5b62b5a8ff 2024-11-19T17:14:12,434 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/9f2217e83a6b4ba1a876ea683ac9df49 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/9f2217e83a6b4ba1a876ea683ac9df49 2024-11-19T17:14:12,435 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/489e641fd1e8449c94379a69f7c81fd8 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/489e641fd1e8449c94379a69f7c81fd8 2024-11-19T17:14:12,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/6c7ac8a6cb614729bef8c602d6eb5275 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/6c7ac8a6cb614729bef8c602d6eb5275 2024-11-19T17:14:12,437 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/edcb6f33a4a3465599e144a1c7fad27a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/edcb6f33a4a3465599e144a1c7fad27a 2024-11-19T17:14:12,438 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/aa7d720c06e44a8d87af784b9ff7064f to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/aa7d720c06e44a8d87af784b9ff7064f 2024-11-19T17:14:12,439 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/c2349cf18af241ab8559cbbfd4192a63 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/c2349cf18af241ab8559cbbfd4192a63 2024-11-19T17:14:12,441 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/2d216efd1aa4404e87ee0ea4d56d6923 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/2d216efd1aa4404e87ee0ea4d56d6923 2024-11-19T17:14:12,442 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/2697b17a65034698b21227949bc17a7c to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/2697b17a65034698b21227949bc17a7c 2024-11-19T17:14:12,443 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/c01ab535b5e649608ed82fa436f5b38c to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/c01ab535b5e649608ed82fa436f5b38c 2024-11-19T17:14:12,444 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/34d9471bddd84730ac1f342ef5a219ed to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/34d9471bddd84730ac1f342ef5a219ed 2024-11-19T17:14:12,445 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/7935e682368a45e39fab7c52c1ba5a6c to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/7935e682368a45e39fab7c52c1ba5a6c 2024-11-19T17:14:12,446 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/03e2261148df447db2e791124dce802c to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/03e2261148df447db2e791124dce802c 2024-11-19T17:14:12,447 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/4d070322eac742eb83a3ecd3a8423c6a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/4d070322eac742eb83a3ecd3a8423c6a 2024-11-19T17:14:12,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/d3b79abd5ab94cf7a475cf741342d458 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/d3b79abd5ab94cf7a475cf741342d458 2024-11-19T17:14:12,449 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/c0bb45ca494c414fa1ad3f6dcda78422 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/c0bb45ca494c414fa1ad3f6dcda78422 2024-11-19T17:14:12,450 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/a8f6a14d1996447d88c8ee5fd930ebda to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/a8f6a14d1996447d88c8ee5fd930ebda 2024-11-19T17:14:12,451 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/a31a066021b34294b1929d67671cf008 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/a31a066021b34294b1929d67671cf008 2024-11-19T17:14:12,452 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/3946624d13164aff9de7abd136999c7f to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/3946624d13164aff9de7abd136999c7f 2024-11-19T17:14:12,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/fdb799d488884069931f3e68489f59ce to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/fdb799d488884069931f3e68489f59ce 2024-11-19T17:14:12,455 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/89f67b4d082d4c5ba5c0163bf8cb12ce to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/89f67b4d082d4c5ba5c0163bf8cb12ce 2024-11-19T17:14:12,456 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/464cb1d766404842b15ea4c183019a0f to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/464cb1d766404842b15ea4c183019a0f 2024-11-19T17:14:12,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/599d715747f140cea68cc837063930bd to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/599d715747f140cea68cc837063930bd 2024-11-19T17:14:12,458 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/b8783eeb558b4c7e930fa136023f9fa2 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/b8783eeb558b4c7e930fa136023f9fa2 2024-11-19T17:14:12,459 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/d78b2cb860d444d682e368f9afa93080 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/d78b2cb860d444d682e368f9afa93080 2024-11-19T17:14:12,460 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/de22ffa6d3444d7e9bc11a68465d7b94 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/de22ffa6d3444d7e9bc11a68465d7b94 2024-11-19T17:14:12,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/21ad9e57b563402a982fad1237461ad8 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/21ad9e57b563402a982fad1237461ad8 2024-11-19T17:14:12,462 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/afd6cb8e7d6a445081d8656add1ecaa5 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/afd6cb8e7d6a445081d8656add1ecaa5 2024-11-19T17:14:12,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/3ffe66af02894054a0debdf857ef87ca to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/3ffe66af02894054a0debdf857ef87ca 2024-11-19T17:14:12,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/9042194d67674ff68a81400655588dd4 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/9042194d67674ff68a81400655588dd4 2024-11-19T17:14:12,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/4b1a9e14c4354d3d8a4bf704f7f3fda6 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/4b1a9e14c4354d3d8a4bf704f7f3fda6 2024-11-19T17:14:12,470 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/recovered.edits/632.seqid, newMaxSeqId=632, maxSeqId=1 2024-11-19T17:14:12,472 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. 2024-11-19T17:14:12,473 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 689491498ff99efc87002cc59bc70ef3: Waiting for close lock at 1732036451027Running coprocessor pre-close hooks at 1732036451027Disabling compacts and flushes for region at 1732036451027Disabling writes for close at 1732036451028 (+1 ms)Obtaining lock to block concurrent updates at 1732036451028Preparing flush snapshotting stores in 689491498ff99efc87002cc59bc70ef3 at 1732036451028Finished memstore snapshotting TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732036451028Flushing stores of TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3. at 1732036451029 (+1 ms)Flushing 689491498ff99efc87002cc59bc70ef3/A: creating writer at 1732036451029Flushing 689491498ff99efc87002cc59bc70ef3/A: appending metadata at 1732036451032 (+3 ms)Flushing 689491498ff99efc87002cc59bc70ef3/A: closing flushed file at 1732036451032Flushing 689491498ff99efc87002cc59bc70ef3/B: creating writer at 1732036451441 (+409 ms)Flushing 689491498ff99efc87002cc59bc70ef3/B: appending metadata at 1732036451444 (+3 ms)Flushing 689491498ff99efc87002cc59bc70ef3/B: closing flushed file at 1732036451445 (+1 ms)Flushing 689491498ff99efc87002cc59bc70ef3/C: creating writer at 1732036451854 (+409 ms)Flushing 689491498ff99efc87002cc59bc70ef3/C: appending metadata at 1732036451856 (+2 ms)Flushing 689491498ff99efc87002cc59bc70ef3/C: closing flushed file at 1732036451856Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b4464dd: reopening flushed file at 1732036452266 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36582fde: reopening flushed file at 1732036452270 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e6999a9: reopening flushed file at 1732036452275 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 689491498ff99efc87002cc59bc70ef3 in 1252ms, sequenceid=629, compaction requested=true at 1732036452280 (+5 ms)Writing region close event to WAL at 1732036452466 (+186 ms)Running coprocessor post-close hooks at 1732036452470 (+4 ms)Closed at 1732036452472 (+2 ms) 2024-11-19T17:14:12,475 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 689491498ff99efc87002cc59bc70ef3 2024-11-19T17:14:12,476 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=689491498ff99efc87002cc59bc70ef3, regionState=CLOSED 2024-11-19T17:14:12,477 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 because future has completed 2024-11-19T17:14:12,480 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-19T17:14:12,480 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 689491498ff99efc87002cc59bc70ef3, server=280a42d09ec8,33873,1732036422473 in 1.6080 sec 2024-11-19T17:14:12,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-19T17:14:12,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=689491498ff99efc87002cc59bc70ef3, UNASSIGN in 1.6150 sec 2024-11-19T17:14:12,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-19T17:14:12,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6210 sec 2024-11-19T17:14:12,485 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732036452485"}]},"ts":"1732036452485"} 2024-11-19T17:14:12,486 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T17:14:12,486 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T17:14:12,488 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6360 sec 2024-11-19T17:14:13,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-19T17:14:13,004 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:14:13,008 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T17:14:13,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:14:13,013 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=37, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:14:13,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-19T17:14:13,015 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=37, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:14:13,019 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3 2024-11-19T17:14:13,023 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A, FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B, FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C, FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/recovered.edits] 2024-11-19T17:14:13,027 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/2337f4eb623140fcb40258ae7dfb843d to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/2337f4eb623140fcb40258ae7dfb843d 2024-11-19T17:14:13,028 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/26130a9d43784ca48ea3db16542fc207 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/26130a9d43784ca48ea3db16542fc207 2024-11-19T17:14:13,029 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/6a7ad2fc5e2d469d96e62dd60ecdbc59 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/A/6a7ad2fc5e2d469d96e62dd60ecdbc59 2024-11-19T17:14:13,032 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/83d5daadbc504e75b43f263a1451635d to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/83d5daadbc504e75b43f263a1451635d 2024-11-19T17:14:13,033 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/856be8a343f94645bd05f5b9e0105b65 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/856be8a343f94645bd05f5b9e0105b65 2024-11-19T17:14:13,034 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/b092411092d142beb774f60a19e937c5 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/B/b092411092d142beb774f60a19e937c5 2024-11-19T17:14:13,037 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/8b42c435f3564e50a015fb00c388871f to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/8b42c435f3564e50a015fb00c388871f 2024-11-19T17:14:13,038 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/bf3ab40cb40e4c2bb8d7f123acaceb49 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/bf3ab40cb40e4c2bb8d7f123acaceb49 2024-11-19T17:14:13,039 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/eec3cee9a9f948e6ac9315b51911c3d4 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/C/eec3cee9a9f948e6ac9315b51911c3d4 2024-11-19T17:14:13,042 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/recovered.edits/632.seqid to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3/recovered.edits/632.seqid 2024-11-19T17:14:13,043 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/689491498ff99efc87002cc59bc70ef3 2024-11-19T17:14:13,043 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T17:14:13,048 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=37, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:14:13,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-19T17:14:13,059 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T17:14:13,100 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T17:14:13,102 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=37, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:14:13,103 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T17:14:13,103 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732036453103"}]},"ts":"9223372036854775807"} 2024-11-19T17:14:13,108 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T17:14:13,108 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 689491498ff99efc87002cc59bc70ef3, NAME => 'TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T17:14:13,109 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T17:14:13,109 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732036453109"}]},"ts":"9223372036854775807"} 2024-11-19T17:14:13,112 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T17:14:13,113 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=37, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:14:13,115 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 106 msec 2024-11-19T17:14:13,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-19T17:14:13,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T17:14:13,125 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:14:13,143 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=242 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_569709797_22 at /127.0.0.1:34612 [Waiting for operation #45] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;280a42d09ec8:33873-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_569709797_22 at /127.0.0.1:41722 [Waiting for operation #433] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-873178183_22 at /127.0.0.1:41558 [Waiting for operation #446] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=344 (was 317) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7381 (was 7196) - AvailableMemoryMB LEAK? - 2024-11-19T17:14:13,154 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=242, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=344, ProcessCount=11, AvailableMemoryMB=7378 2024-11-19T17:14:13,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T17:14:13,157 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T17:14:13,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T17:14:13,159 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=38, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T17:14:13,159 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:13,159 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 38 2024-11-19T17:14:13,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-19T17:14:13,161 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=38, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T17:14:13,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741985_1161 (size=1061) 2024-11-19T17:14:13,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-19T17:14:13,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-19T17:14:13,569 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e 2024-11-19T17:14:13,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741986_1162 (size=53) 2024-11-19T17:14:13,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-19T17:14:13,975 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:14:13,976 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5d916b1a7715625884544afede338b7a, disabling compactions & flushes 2024-11-19T17:14:13,976 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:13,976 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:13,976 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. after waiting 0 ms 2024-11-19T17:14:13,976 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:13,976 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:13,976 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5d916b1a7715625884544afede338b7a: Waiting for close lock at 1732036453975Disabling compacts and flushes for region at 1732036453975Disabling writes for close at 1732036453976 (+1 ms)Writing region close event to WAL at 1732036453976Closed at 1732036453976 2024-11-19T17:14:13,977 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=38, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T17:14:13,977 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732036453977"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732036453977"}]},"ts":"1732036453977"} 2024-11-19T17:14:13,979 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T17:14:13,980 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=38, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T17:14:13,981 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732036453980"}]},"ts":"1732036453980"} 2024-11-19T17:14:13,982 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T17:14:13,982 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d916b1a7715625884544afede338b7a, ASSIGN}] 2024-11-19T17:14:13,983 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d916b1a7715625884544afede338b7a, ASSIGN 2024-11-19T17:14:13,984 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d916b1a7715625884544afede338b7a, ASSIGN; state=OFFLINE, location=280a42d09ec8,33873,1732036422473; forceNewPlan=false, retain=false 2024-11-19T17:14:14,135 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=5d916b1a7715625884544afede338b7a, regionState=OPENING, regionLocation=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:14,137 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d916b1a7715625884544afede338b7a, ASSIGN because future has completed 2024-11-19T17:14:14,137 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473}] 2024-11-19T17:14:14,293 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:14,293 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:14:14,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-19T17:14:14,293 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:14,294 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:14:14,294 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:14,294 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:14,295 INFO [StoreOpener-5d916b1a7715625884544afede338b7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:14,296 INFO [StoreOpener-5d916b1a7715625884544afede338b7a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T17:14:14,297 INFO [StoreOpener-5d916b1a7715625884544afede338b7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5d916b1a7715625884544afede338b7a columnFamilyName A 2024-11-19T17:14:14,297 DEBUG [StoreOpener-5d916b1a7715625884544afede338b7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:14,298 INFO [StoreOpener-5d916b1a7715625884544afede338b7a-1 {}] regionserver.HStore(327): Store=5d916b1a7715625884544afede338b7a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:14:14,298 INFO [StoreOpener-5d916b1a7715625884544afede338b7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:14,299 INFO [StoreOpener-5d916b1a7715625884544afede338b7a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T17:14:14,299 INFO [StoreOpener-5d916b1a7715625884544afede338b7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5d916b1a7715625884544afede338b7a columnFamilyName B 2024-11-19T17:14:14,299 DEBUG [StoreOpener-5d916b1a7715625884544afede338b7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:14,300 INFO [StoreOpener-5d916b1a7715625884544afede338b7a-1 {}] regionserver.HStore(327): Store=5d916b1a7715625884544afede338b7a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:14:14,300 INFO [StoreOpener-5d916b1a7715625884544afede338b7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:14,300 INFO [StoreOpener-5d916b1a7715625884544afede338b7a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T17:14:14,301 INFO [StoreOpener-5d916b1a7715625884544afede338b7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5d916b1a7715625884544afede338b7a columnFamilyName C 2024-11-19T17:14:14,301 DEBUG [StoreOpener-5d916b1a7715625884544afede338b7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:14,301 INFO [StoreOpener-5d916b1a7715625884544afede338b7a-1 {}] regionserver.HStore(327): Store=5d916b1a7715625884544afede338b7a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:14:14,301 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:14,301 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:14,302 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a 2024-11-19T17:14:14,302 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a 2024-11-19T17:14:14,303 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:14,303 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:14,304 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T17:14:14,305 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:14,307 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T17:14:14,307 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 5d916b1a7715625884544afede338b7a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64035136, jitterRate=-0.04580211639404297}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T17:14:14,307 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:14,308 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 5d916b1a7715625884544afede338b7a: Running coprocessor pre-open hook at 1732036454294Writing region info on filesystem at 1732036454294Initializing all the Stores at 1732036454295 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036454295Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036454295Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036454295Cleaning up temporary data from old regions at 1732036454303 (+8 ms)Running coprocessor post-open hooks at 1732036454307 (+4 ms)Region opened successfully at 1732036454308 (+1 ms) 2024-11-19T17:14:14,309 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., pid=40, masterSystemTime=1732036454289 2024-11-19T17:14:14,311 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:14,311 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:14,311 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=5d916b1a7715625884544afede338b7a, regionState=OPEN, openSeqNum=2, regionLocation=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:14,313 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 because future has completed 2024-11-19T17:14:14,315 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-19T17:14:14,315 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure 5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 in 177 msec 2024-11-19T17:14:14,317 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-19T17:14:14,317 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d916b1a7715625884544afede338b7a, ASSIGN in 333 msec 2024-11-19T17:14:14,318 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=38, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T17:14:14,318 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732036454318"}]},"ts":"1732036454318"} 2024-11-19T17:14:14,320 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T17:14:14,321 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=38, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T17:14:14,322 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1640 sec 2024-11-19T17:14:15,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-19T17:14:15,305 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:14:15,306 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79527845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:15,306 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:14:15,306 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:14:15,308 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:14:15,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:14:15,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:14:15,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d3d65be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:15,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:14:15,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:14:15,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:15,310 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41116, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:14:15,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e548935, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:15,311 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:14:15,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:14:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:14:15,314 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:14:15,315 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T17:14:15,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 280a42d09ec8,40603,1732036421715 2024-11-19T17:14:15,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@67a4aeb8 2024-11-19T17:14:15,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T17:14:15,317 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T17:14:15,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T17:14:15,322 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T17:14:15,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=41, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T17:14:15,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741987_1163 (size=1097) 2024-11-19T17:14:15,741 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-19T17:14:15,741 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-19T17:14:15,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T17:14:15,752 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d916b1a7715625884544afede338b7a, REOPEN/MOVE}] 2024-11-19T17:14:15,753 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d916b1a7715625884544afede338b7a, REOPEN/MOVE 2024-11-19T17:14:15,754 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=43 updating hbase:meta row=5d916b1a7715625884544afede338b7a, regionState=CLOSING, regionLocation=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:15,754 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40603 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=CLOSING, location=280a42d09ec8,33873,1732036422473, table=TestAcidGuarantees, region=5d916b1a7715625884544afede338b7a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T17:14:15,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d916b1a7715625884544afede338b7a, REOPEN/MOVE because future has completed 2024-11-19T17:14:15,756 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T17:14:15,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473}] 2024-11-19T17:14:15,908 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] handler.UnassignRegionHandler(122): Close 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:15,909 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T17:14:15,909 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1722): Closing 5d916b1a7715625884544afede338b7a, disabling compactions & flushes 2024-11-19T17:14:15,909 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:15,909 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:15,909 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. after waiting 0 ms 2024-11-19T17:14:15,909 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:15,913 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T17:14:15,913 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:15,914 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1676): Region close journal for 5d916b1a7715625884544afede338b7a: Waiting for close lock at 1732036455909Running coprocessor pre-close hooks at 1732036455909Disabling compacts and flushes for region at 1732036455909Disabling writes for close at 1732036455909Writing region close event to WAL at 1732036455909Running coprocessor post-close hooks at 1732036455913 (+4 ms)Closed at 1732036455913 2024-11-19T17:14:15,914 WARN [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegionServer(3299): Not adding moved region record: 5d916b1a7715625884544afede338b7a to self. 2024-11-19T17:14:15,916 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] handler.UnassignRegionHandler(157): Closed 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:15,916 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=43 updating hbase:meta row=5d916b1a7715625884544afede338b7a, regionState=CLOSED 2024-11-19T17:14:15,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 because future has completed 2024-11-19T17:14:15,921 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=43 2024-11-19T17:14:15,921 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=43, state=SUCCESS, hasLock=false; CloseRegionProcedure 5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 in 163 msec 2024-11-19T17:14:15,921 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d916b1a7715625884544afede338b7a, REOPEN/MOVE; state=CLOSED, location=280a42d09ec8,33873,1732036422473; forceNewPlan=false, retain=true 2024-11-19T17:14:16,072 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=43 updating hbase:meta row=5d916b1a7715625884544afede338b7a, regionState=OPENING, regionLocation=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:16,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d916b1a7715625884544afede338b7a, REOPEN/MOVE because future has completed 2024-11-19T17:14:16,074 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=43, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473}] 2024-11-19T17:14:16,229 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:16,229 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:14:16,230 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:16,230 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:14:16,230 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:16,230 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:16,234 INFO [StoreOpener-5d916b1a7715625884544afede338b7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:16,235 INFO [StoreOpener-5d916b1a7715625884544afede338b7a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T17:14:16,240 INFO [StoreOpener-5d916b1a7715625884544afede338b7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5d916b1a7715625884544afede338b7a columnFamilyName A 2024-11-19T17:14:16,242 DEBUG [StoreOpener-5d916b1a7715625884544afede338b7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:16,242 INFO [StoreOpener-5d916b1a7715625884544afede338b7a-1 {}] regionserver.HStore(327): Store=5d916b1a7715625884544afede338b7a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:14:16,243 INFO [StoreOpener-5d916b1a7715625884544afede338b7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:16,243 INFO [StoreOpener-5d916b1a7715625884544afede338b7a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T17:14:16,244 INFO [StoreOpener-5d916b1a7715625884544afede338b7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5d916b1a7715625884544afede338b7a columnFamilyName B 2024-11-19T17:14:16,244 DEBUG [StoreOpener-5d916b1a7715625884544afede338b7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:16,244 INFO [StoreOpener-5d916b1a7715625884544afede338b7a-1 {}] regionserver.HStore(327): Store=5d916b1a7715625884544afede338b7a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:14:16,244 INFO [StoreOpener-5d916b1a7715625884544afede338b7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:16,245 INFO [StoreOpener-5d916b1a7715625884544afede338b7a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T17:14:16,245 INFO [StoreOpener-5d916b1a7715625884544afede338b7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5d916b1a7715625884544afede338b7a columnFamilyName C 2024-11-19T17:14:16,245 DEBUG [StoreOpener-5d916b1a7715625884544afede338b7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:16,245 INFO [StoreOpener-5d916b1a7715625884544afede338b7a-1 {}] regionserver.HStore(327): Store=5d916b1a7715625884544afede338b7a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:14:16,246 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:16,246 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:16,246 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a 2024-11-19T17:14:16,247 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a 2024-11-19T17:14:16,248 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:16,248 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:16,248 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T17:14:16,250 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:16,250 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 5d916b1a7715625884544afede338b7a; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67632658, jitterRate=0.007805138826370239}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T17:14:16,250 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:16,252 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 5d916b1a7715625884544afede338b7a: Running coprocessor pre-open hook at 1732036456230Writing region info on filesystem at 1732036456230Initializing all the Stores at 1732036456231 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036456231Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036456231Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036456231Cleaning up temporary data from old regions at 1732036456248 (+17 ms)Running coprocessor post-open hooks at 1732036456250 (+2 ms)Region opened successfully at 1732036456252 (+2 ms) 2024-11-19T17:14:16,252 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., pid=45, masterSystemTime=1732036456226 2024-11-19T17:14:16,254 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:16,254 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:16,255 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=43 updating hbase:meta row=5d916b1a7715625884544afede338b7a, regionState=OPEN, openSeqNum=5, regionLocation=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:16,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=43, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 because future has completed 2024-11-19T17:14:16,258 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=43 2024-11-19T17:14:16,258 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=43, state=SUCCESS, hasLock=false; OpenRegionProcedure 5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 in 183 msec 2024-11-19T17:14:16,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-19T17:14:16,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d916b1a7715625884544afede338b7a, REOPEN/MOVE in 506 msec 2024-11-19T17:14:16,263 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=41 2024-11-19T17:14:16,263 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=41, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 516 msec 2024-11-19T17:14:16,266 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 940 msec 2024-11-19T17:14:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=41 2024-11-19T17:14:16,268 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:14:16,269 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adc4e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:16,269 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:14:16,269 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:14:16,270 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:14:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:14:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:14:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@697e8d9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:14:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:14:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:16,272 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:14:16,272 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38c7cf29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:16,273 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681c9b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:16,273 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:14:16,273 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:14:16,274 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:14:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:14:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:14:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6455e7c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:14:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:14:16,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:16,275 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:14:16,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32346c86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:16,277 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d91e8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:16,277 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:14:16,277 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:14:16,277 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:14:16,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:14:16,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:14:16,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cc378bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:16,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:14:16,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:14:16,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:16,279 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:14:16,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@638f174a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:16,280 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771f5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:16,280 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:14:16,280 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:14:16,281 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:14:16,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:14:16,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:14:16,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6385c816, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:16,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:14:16,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:14:16,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:16,282 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:14:16,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@590208c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:16,283 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b75e2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:16,283 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:14:16,283 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:14:16,284 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:14:16,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:14:16,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:14:16,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1adeee8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:16,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:14:16,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:14:16,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:16,285 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:14:16,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e957eb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:16,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5453a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:16,286 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:14:16,287 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:14:16,287 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:14:16,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:14:16,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:14:16,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10ceaee0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:16,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:14:16,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:14:16,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:16,288 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:14:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56112bd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:16,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b6e690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:16,290 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:14:16,290 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:14:16,290 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:14:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:14:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:14:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2014590d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:14:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:14:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:16,291 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:14:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@448192b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:16,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:16,292 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:14:16,292 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:14:16,293 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:14:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:14:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:14:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71441391, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:14:16,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:14:16,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:16,294 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:14:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50c3ce6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:16,295 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dff183a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:16,295 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:14:16,295 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:14:16,296 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:14:16,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:14:16,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:14:16,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2df55c6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:16,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:14:16,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:14:16,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:16,297 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:14:16,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@273813dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:16,299 DEBUG [Thread-789 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:14:16,299 DEBUG [Thread-794 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:14:16,299 DEBUG [Thread-798 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:14:16,299 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:14:16,299 DEBUG [Thread-792 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:14:16,299 DEBUG [Thread-787 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:14:16,299 DEBUG [Thread-781 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:14:16,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:14:16,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:14:16,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:14:16,300 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:14:16,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:14:16,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:14:16,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:14:16,300 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:14:16,301 DEBUG [Thread-783 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:14:16,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:14:16,301 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:14:16,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-19T17:14:16,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:14:16,301 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:14:16,301 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:14:16,301 DEBUG [Thread-785 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:14:16,302 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:14:16,302 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:14:16,302 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:14:16,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:14:16,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:14:16,304 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:14:16,304 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:14:16,305 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:14:16,305 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:14:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T17:14:16,306 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:14:16,306 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:14:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:14:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:14:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5] 2024-11-19T17:14:16,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5] 2024-11-19T17:14:16,308 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34222, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:14:16,309 DEBUG [Thread-796 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:14:16,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5] 2024-11-19T17:14:16,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:14:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:14:16,311 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:14:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5] 2024-11-19T17:14:16,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5] 2024-11-19T17:14:16,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5] 2024-11-19T17:14:16,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5] 2024-11-19T17:14:16,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5] 2024-11-19T17:14:16,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5] 2024-11-19T17:14:16,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:16,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d916b1a7715625884544afede338b7a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T17:14:16,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=A 2024-11-19T17:14:16,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:16,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=B 2024-11-19T17:14:16,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:16,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=C 2024-11-19T17:14:16,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:16,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b98eaf2679904e98982d1460426cda10_5d916b1a7715625884544afede338b7a is 50, key is test_row_0/A:col10/1732036456324/Put/seqid=0 2024-11-19T17:14:16,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732036456347, max timestamp=1732036456378Num uniques 859; 2024-11-19T17:14:16,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T17:14:16,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036456362, max timestamp=1732036456378Num uniques 859; 2024-11-19T17:14:16,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:14:16,396 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732036456362, max timestamp=1732036456386Num uniques 859; 2024-11-19T17:14:16,396 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-19T17:14:16,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741988_1164 (size=19474) 2024-11-19T17:14:16,411 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T17:14:16,418 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b98eaf2679904e98982d1460426cda10_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b98eaf2679904e98982d1460426cda10_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:16,420 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/f80dd0aeeb17414db815143a1979b3a0, store: [table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:16,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/f80dd0aeeb17414db815143a1979b3a0 is 175, key is test_row_0/A:col10/1732036456324/Put/seqid=0 2024-11-19T17:14:16,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741989_1165 (size=56733) 2024-11-19T17:14:16,448 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/f80dd0aeeb17414db815143a1979b3a0 2024-11-19T17:14:16,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:16,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036516431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:16,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:16,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036516431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:16,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:16,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036516433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:16,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:16,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:16,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:16,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:16,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:16,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:16,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:16,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036516434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:16,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:16,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:16,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:16,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:16,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:16,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:16,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:16,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036516432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:16,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:16,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:16,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:16,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T17:14:16,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:16,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:16,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:16,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:16,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:16,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:16,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/5ad25038d8974686b2c4c71a087382f2 is 50, key is test_row_0/B:col10/1732036456324/Put/seqid=0 2024-11-19T17:14:16,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741990_1166 (size=12001) 2024-11-19T17:14:16,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:16,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036516564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:16,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:16,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036516565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:16,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:16,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:16,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:16,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:16,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036516565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:16,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:16,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:16,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:16,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:16,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:16,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036516565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:16,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:16,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:16,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:16,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:16,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036516565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:16,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:16,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:16,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:16,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:16,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:16,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T17:14:16,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:16,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:16,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:16,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:16,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:16,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:16,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T17:14:16,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T17:14:16,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:16,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:16,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:16,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:16,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:16,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:16,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:16,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036516773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:16,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:16,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036516774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:16,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:16,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:16,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:16,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036516774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:16,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:16,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:16,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036516774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:16,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:16,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036516774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:16,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:16,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:16,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:16,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:16,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:16,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:16,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:16,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:16,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:16,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/5ad25038d8974686b2c4c71a087382f2 2024-11-19T17:14:16,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T17:14:16,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:16,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:16,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:16,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:16,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:16,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:16,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/b449ac56500d42b890de2fa65c9d76e8 is 50, key is test_row_0/C:col10/1732036456324/Put/seqid=0 2024-11-19T17:14:16,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T17:14:16,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741991_1167 (size=12001) 2024-11-19T17:14:16,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/b449ac56500d42b890de2fa65c9d76e8 2024-11-19T17:14:16,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/f80dd0aeeb17414db815143a1979b3a0 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/f80dd0aeeb17414db815143a1979b3a0 2024-11-19T17:14:16,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/f80dd0aeeb17414db815143a1979b3a0, entries=300, sequenceid=15, filesize=55.4 K 2024-11-19T17:14:16,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/5ad25038d8974686b2c4c71a087382f2 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/5ad25038d8974686b2c4c71a087382f2 2024-11-19T17:14:16,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/5ad25038d8974686b2c4c71a087382f2, entries=150, sequenceid=15, filesize=11.7 K 2024-11-19T17:14:16,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/b449ac56500d42b890de2fa65c9d76e8 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/b449ac56500d42b890de2fa65c9d76e8 2024-11-19T17:14:16,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/b449ac56500d42b890de2fa65c9d76e8, entries=150, sequenceid=15, filesize=11.7 K 2024-11-19T17:14:16,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.31 KB/132415 for 5d916b1a7715625884544afede338b7a in 647ms, sequenceid=15, compaction requested=false 2024-11-19T17:14:16,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:17,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T17:14:17,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:17,077 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 5d916b1a7715625884544afede338b7a 3/3 column families, dataSize=129.31 KB heapSize=364.54 KB 2024-11-19T17:14:17,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=A 2024-11-19T17:14:17,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:17,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=B 2024-11-19T17:14:17,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:17,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=C 2024-11-19T17:14:17,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:17,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:17,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:17,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196ad0bc1681f442b090dc01c33a4d1437_5d916b1a7715625884544afede338b7a is 50, key is test_row_0/A:col10/1732036456431/Put/seqid=0 2024-11-19T17:14:17,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741992_1168 (size=12154) 2024-11-19T17:14:17,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:17,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:17,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036517099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036517101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:17,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:17,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036517103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:17,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:17,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:17,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:17,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036517106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:17,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:17,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036517109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:17,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:17,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:17,115 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196ad0bc1681f442b090dc01c33a4d1437_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196ad0bc1681f442b090dc01c33a4d1437_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:17,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/6026ce3e3f3a4dd28e1f25e803c3db6d, store: [table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:17,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/6026ce3e3f3a4dd28e1f25e803c3db6d is 175, key is test_row_0/A:col10/1732036456431/Put/seqid=0 2024-11-19T17:14:17,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741993_1169 (size=30955) 2024-11-19T17:14:17,133 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/6026ce3e3f3a4dd28e1f25e803c3db6d 2024-11-19T17:14:17,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/58b2dfc76cc3431fad0db6e68eb0b706 is 50, key is test_row_0/B:col10/1732036456431/Put/seqid=0 2024-11-19T17:14:17,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741994_1170 (size=12001) 2024-11-19T17:14:17,168 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/58b2dfc76cc3431fad0db6e68eb0b706 2024-11-19T17:14:17,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/a7fbebc0be6842a78026dbe20a32a075 is 50, key is test_row_0/C:col10/1732036456431/Put/seqid=0 2024-11-19T17:14:17,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741995_1171 (size=12001) 2024-11-19T17:14:17,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:17,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:17,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036517215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:17,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036517214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:17,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:17,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036517215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:17,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:17,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:17,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:17,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:17,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036517224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:17,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:17,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036517224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:17,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:17,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:17,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:17,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036517423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:17,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:17,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036517423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:17,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:17,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:17,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:17,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036517423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:17,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:17,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:17,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036517434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:17,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:17,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:17,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036517434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:17,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:17,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T17:14:17,600 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.86 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/a7fbebc0be6842a78026dbe20a32a075 2024-11-19T17:14:17,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/6026ce3e3f3a4dd28e1f25e803c3db6d as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/6026ce3e3f3a4dd28e1f25e803c3db6d 2024-11-19T17:14:17,613 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/6026ce3e3f3a4dd28e1f25e803c3db6d, entries=150, sequenceid=50, filesize=30.2 K 2024-11-19T17:14:17,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/58b2dfc76cc3431fad0db6e68eb0b706 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/58b2dfc76cc3431fad0db6e68eb0b706 2024-11-19T17:14:17,621 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/58b2dfc76cc3431fad0db6e68eb0b706, entries=150, sequenceid=50, filesize=11.7 K 2024-11-19T17:14:17,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/a7fbebc0be6842a78026dbe20a32a075 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/a7fbebc0be6842a78026dbe20a32a075 2024-11-19T17:14:17,632 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/a7fbebc0be6842a78026dbe20a32a075, entries=150, sequenceid=50, filesize=11.7 K 2024-11-19T17:14:17,633 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~129.31 KB/132415, heapSize ~364.49 KB/373240, currentSize=53.67 KB/54960 for 5d916b1a7715625884544afede338b7a in 556ms, sequenceid=50, compaction requested=false 2024-11-19T17:14:17,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:17,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:17,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-19T17:14:17,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-19T17:14:17,640 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-19T17:14:17,640 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3290 sec 2024-11-19T17:14:17,643 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.3420 sec 2024-11-19T17:14:17,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:17,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d916b1a7715625884544afede338b7a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T17:14:17,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=A 2024-11-19T17:14:17,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:17,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=B 2024-11-19T17:14:17,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:17,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=C 2024-11-19T17:14:17,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:17,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119fe0401679a7b484e9ee33888c6d66a89_5d916b1a7715625884544afede338b7a is 50, key is test_row_0/A:col10/1732036457737/Put/seqid=0 2024-11-19T17:14:17,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741996_1172 (size=12154) 2024-11-19T17:14:17,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:17,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036517790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:17,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:17,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:17,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036517792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:17,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:17,797 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:17,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:17,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036517792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:17,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036517795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:17,803 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119fe0401679a7b484e9ee33888c6d66a89_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119fe0401679a7b484e9ee33888c6d66a89_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:17,804 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/97ce65aa269a4823936ed378f4482192, store: [table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:17,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/97ce65aa269a4823936ed378f4482192 is 175, key is test_row_0/A:col10/1732036457737/Put/seqid=0 2024-11-19T17:14:17,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036517797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:17,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741997_1173 (size=30955) 2024-11-19T17:14:17,825 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/97ce65aa269a4823936ed378f4482192 2024-11-19T17:14:17,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/27061b2342ca45bba50b2ab40d5a2614 is 50, key is test_row_0/B:col10/1732036457737/Put/seqid=0 2024-11-19T17:14:17,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741998_1174 (size=12001) 2024-11-19T17:14:17,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:17,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036517904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:17,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:17,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:17,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036517904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:17,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:17,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:17,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036517904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:17,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:17,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036517904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:17,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:17,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:17,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036517914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:18,018 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T17:14:18,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:18,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036518113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:18,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:18,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036518114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:18,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:18,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036518114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:18,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:18,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036518114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:18,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:18,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:18,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:18,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036518124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:18,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:18,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/27061b2342ca45bba50b2ab40d5a2614 2024-11-19T17:14:18,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/c0123e5e1d50477b81bc76d0b5f3b2e5 is 50, key is test_row_0/C:col10/1732036457737/Put/seqid=0 2024-11-19T17:14:18,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741999_1175 (size=12001) 2024-11-19T17:14:18,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/c0123e5e1d50477b81bc76d0b5f3b2e5 2024-11-19T17:14:18,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/97ce65aa269a4823936ed378f4482192 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/97ce65aa269a4823936ed378f4482192 2024-11-19T17:14:18,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/97ce65aa269a4823936ed378f4482192, entries=150, sequenceid=63, filesize=30.2 K 2024-11-19T17:14:18,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/27061b2342ca45bba50b2ab40d5a2614 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/27061b2342ca45bba50b2ab40d5a2614 2024-11-19T17:14:18,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/27061b2342ca45bba50b2ab40d5a2614, entries=150, sequenceid=63, filesize=11.7 K 2024-11-19T17:14:18,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/c0123e5e1d50477b81bc76d0b5f3b2e5 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/c0123e5e1d50477b81bc76d0b5f3b2e5 2024-11-19T17:14:18,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/c0123e5e1d50477b81bc76d0b5f3b2e5, entries=150, sequenceid=63, filesize=11.7 K 2024-11-19T17:14:18,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 5d916b1a7715625884544afede338b7a in 558ms, sequenceid=63, compaction requested=true 2024-11-19T17:14:18,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:18,296 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:18,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d916b1a7715625884544afede338b7a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:14:18,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:18,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d916b1a7715625884544afede338b7a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:14:18,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:18,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d916b1a7715625884544afede338b7a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:14:18,297 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:18,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:18,299 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:18,299 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 5d916b1a7715625884544afede338b7a/A is initiating minor compaction (all files) 2024-11-19T17:14:18,300 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d916b1a7715625884544afede338b7a/A in TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:18,300 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/f80dd0aeeb17414db815143a1979b3a0, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/6026ce3e3f3a4dd28e1f25e803c3db6d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/97ce65aa269a4823936ed378f4482192] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp, totalSize=115.9 K 2024-11-19T17:14:18,300 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:18,300 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. files: [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/f80dd0aeeb17414db815143a1979b3a0, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/6026ce3e3f3a4dd28e1f25e803c3db6d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/97ce65aa269a4823936ed378f4482192] 2024-11-19T17:14:18,301 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:18,301 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 5d916b1a7715625884544afede338b7a/B is initiating minor compaction (all files) 2024-11-19T17:14:18,301 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d916b1a7715625884544afede338b7a/B in TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:18,301 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting f80dd0aeeb17414db815143a1979b3a0, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732036456308 2024-11-19T17:14:18,301 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/5ad25038d8974686b2c4c71a087382f2, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/58b2dfc76cc3431fad0db6e68eb0b706, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/27061b2342ca45bba50b2ab40d5a2614] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp, totalSize=35.2 K 2024-11-19T17:14:18,302 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ad25038d8974686b2c4c71a087382f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732036456322 2024-11-19T17:14:18,302 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 6026ce3e3f3a4dd28e1f25e803c3db6d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732036456410 2024-11-19T17:14:18,303 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58b2dfc76cc3431fad0db6e68eb0b706, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732036456410 2024-11-19T17:14:18,303 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 97ce65aa269a4823936ed378f4482192, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732036457098 2024-11-19T17:14:18,304 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27061b2342ca45bba50b2ab40d5a2614, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732036457098 2024-11-19T17:14:18,318 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d916b1a7715625884544afede338b7a#B#compaction#157 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:18,318 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/b599280704774080bf46549ff45c5889 is 50, key is test_row_0/B:col10/1732036457737/Put/seqid=0 2024-11-19T17:14:18,319 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:18,324 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411196fb11c9b31c647f49c722fcec4ec8cf7_5d916b1a7715625884544afede338b7a store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:18,329 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411196fb11c9b31c647f49c722fcec4ec8cf7_5d916b1a7715625884544afede338b7a, store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:18,330 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196fb11c9b31c647f49c722fcec4ec8cf7_5d916b1a7715625884544afede338b7a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:18,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742000_1176 (size=12104) 2024-11-19T17:14:18,354 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/b599280704774080bf46549ff45c5889 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/b599280704774080bf46549ff45c5889 2024-11-19T17:14:18,369 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d916b1a7715625884544afede338b7a/B of 5d916b1a7715625884544afede338b7a into b599280704774080bf46549ff45c5889(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:18,369 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:18,369 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., storeName=5d916b1a7715625884544afede338b7a/B, priority=13, startTime=1732036458297; duration=0sec 2024-11-19T17:14:18,369 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:18,369 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d916b1a7715625884544afede338b7a:B 2024-11-19T17:14:18,369 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:18,371 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:18,371 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 5d916b1a7715625884544afede338b7a/C is initiating minor compaction (all files) 2024-11-19T17:14:18,371 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d916b1a7715625884544afede338b7a/C in TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:18,371 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/b449ac56500d42b890de2fa65c9d76e8, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/a7fbebc0be6842a78026dbe20a32a075, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/c0123e5e1d50477b81bc76d0b5f3b2e5] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp, totalSize=35.2 K 2024-11-19T17:14:18,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742001_1177 (size=4469) 2024-11-19T17:14:18,372 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting b449ac56500d42b890de2fa65c9d76e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732036456322 2024-11-19T17:14:18,373 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d916b1a7715625884544afede338b7a#A#compaction#158 average throughput is 0.46 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:18,373 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/ec74e8e9e56a47ed8e736ed046620633 is 175, key is test_row_0/A:col10/1732036457737/Put/seqid=0 2024-11-19T17:14:18,374 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7fbebc0be6842a78026dbe20a32a075, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732036456410 2024-11-19T17:14:18,376 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0123e5e1d50477b81bc76d0b5f3b2e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732036457098 2024-11-19T17:14:18,385 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d916b1a7715625884544afede338b7a#C#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:18,386 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/923dea8eb3e348e59cbd56a21d29260f is 50, key is test_row_0/C:col10/1732036457737/Put/seqid=0 2024-11-19T17:14:18,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742002_1178 (size=31058) 2024-11-19T17:14:18,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742003_1179 (size=12104) 2024-11-19T17:14:18,403 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/923dea8eb3e348e59cbd56a21d29260f as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/923dea8eb3e348e59cbd56a21d29260f 2024-11-19T17:14:18,411 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d916b1a7715625884544afede338b7a/C of 5d916b1a7715625884544afede338b7a into 923dea8eb3e348e59cbd56a21d29260f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:18,411 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:18,411 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., storeName=5d916b1a7715625884544afede338b7a/C, priority=13, startTime=1732036458297; duration=0sec 2024-11-19T17:14:18,411 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:18,411 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d916b1a7715625884544afede338b7a:C 2024-11-19T17:14:18,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:18,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d916b1a7715625884544afede338b7a 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-19T17:14:18,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=A 2024-11-19T17:14:18,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:18,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=B 2024-11-19T17:14:18,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036457789, max timestamp=1732036458425Num uniques 859; 2024-11-19T17:14:18,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:18,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=C 2024-11-19T17:14:18,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:14:18,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:18,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119972bec70d67c4d59b2d6454f5f702266_5d916b1a7715625884544afede338b7a is 50, key is test_row_0/A:col10/1732036458424/Put/seqid=0 2024-11-19T17:14:18,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742004_1180 (size=12154) 2024-11-19T17:14:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T17:14:18,464 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:14:18,465 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:14:18,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-19T17:14:18,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T17:14:18,467 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:14:18,468 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:14:18,468 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:14:18,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:18,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036518451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:18,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:18,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036518457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:18,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:18,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:18,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:18,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036518487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:18,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:18,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036518487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:18,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:18,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:18,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:18,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036518487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:18,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:18,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T17:14:18,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036518594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:18,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:18,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036518594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:18,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:18,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:18,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:18,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036518604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:18,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036518605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:18,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036518605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:18,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T17:14:18,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:18,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:18,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:18,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:18,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:18,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:18,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T17:14:18,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:18,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:18,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:18,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:18,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T17:14:18,793 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/ec74e8e9e56a47ed8e736ed046620633 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/ec74e8e9e56a47ed8e736ed046620633 2024-11-19T17:14:18,800 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d916b1a7715625884544afede338b7a/A of 5d916b1a7715625884544afede338b7a into ec74e8e9e56a47ed8e736ed046620633(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:18,800 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:18,800 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., storeName=5d916b1a7715625884544afede338b7a/A, priority=13, startTime=1732036458296; duration=0sec 2024-11-19T17:14:18,800 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:18,800 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d916b1a7715625884544afede338b7a:A 2024-11-19T17:14:18,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:18,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036518804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:18,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036518804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:18,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:18,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036518814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:18,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:18,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036518814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:18,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:18,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:18,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036518814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:18,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:18,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:18,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:18,863 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:18,868 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119972bec70d67c4d59b2d6454f5f702266_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119972bec70d67c4d59b2d6454f5f702266_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:18,869 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/5447bf1cd9c04c9f9ac82074364bf762, store: [table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:18,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/5447bf1cd9c04c9f9ac82074364bf762 is 175, key is test_row_0/A:col10/1732036458424/Put/seqid=0 2024-11-19T17:14:18,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742005_1181 (size=30955) 2024-11-19T17:14:18,882 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=88, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/5447bf1cd9c04c9f9ac82074364bf762 2024-11-19T17:14:18,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/0fc07bf6a56a46658c25d7e8e42c18a7 is 50, key is test_row_0/B:col10/1732036458424/Put/seqid=0 2024-11-19T17:14:18,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742006_1182 (size=12001) 2024-11-19T17:14:18,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/0fc07bf6a56a46658c25d7e8e42c18a7 2024-11-19T17:14:18,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/ec84485fab9d4f26ac59e9bcc869427a is 50, key is test_row_0/C:col10/1732036458424/Put/seqid=0 2024-11-19T17:14:18,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742007_1183 (size=12001) 2024-11-19T17:14:18,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T17:14:18,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:18,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:18,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:18,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:18,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:18,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:19,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T17:14:19,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:19,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:19,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:19,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:19,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T17:14:19,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036519124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:19,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036519124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:19,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:19,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:19,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036519124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:19,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:19,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036519124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:19,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:19,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036519126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:19,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:19,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:19,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:19,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:19,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:19,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:19,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:19,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:19,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:19,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:19,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:19,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:19,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T17:14:19,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:19,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:19,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:19,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:19,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:19,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:19,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/ec84485fab9d4f26ac59e9bcc869427a 2024-11-19T17:14:19,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/5447bf1cd9c04c9f9ac82074364bf762 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/5447bf1cd9c04c9f9ac82074364bf762 2024-11-19T17:14:19,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/5447bf1cd9c04c9f9ac82074364bf762, entries=150, sequenceid=88, filesize=30.2 K 2024-11-19T17:14:19,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/0fc07bf6a56a46658c25d7e8e42c18a7 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/0fc07bf6a56a46658c25d7e8e42c18a7 2024-11-19T17:14:19,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/0fc07bf6a56a46658c25d7e8e42c18a7, entries=150, sequenceid=88, filesize=11.7 K 2024-11-19T17:14:19,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/ec84485fab9d4f26ac59e9bcc869427a as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/ec84485fab9d4f26ac59e9bcc869427a 2024-11-19T17:14:19,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/ec84485fab9d4f26ac59e9bcc869427a, entries=150, sequenceid=88, filesize=11.7 K 2024-11-19T17:14:19,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~102.87 KB/105340, heapSize ~301.91 KB/309152, currentSize=80.51 KB/82440 for 5d916b1a7715625884544afede338b7a in 930ms, sequenceid=88, compaction requested=false 2024-11-19T17:14:19,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:19,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T17:14:19,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:19,394 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 5d916b1a7715625884544afede338b7a 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T17:14:19,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=A 2024-11-19T17:14:19,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:19,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=B 2024-11-19T17:14:19,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:19,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=C 2024-11-19T17:14:19,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:19,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194778a3837ce842d6b519fdaf7369abd6_5d916b1a7715625884544afede338b7a is 50, key is test_row_0/A:col10/1732036458464/Put/seqid=0 2024-11-19T17:14:19,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742008_1184 (size=12154) 2024-11-19T17:14:19,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,428 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194778a3837ce842d6b519fdaf7369abd6_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194778a3837ce842d6b519fdaf7369abd6_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:19,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/b862fad5d9954e35a74e2cd634ec5035, store: [table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:19,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/b862fad5d9954e35a74e2cd634ec5035 is 175, key is test_row_0/A:col10/1732036458464/Put/seqid=0 2024-11-19T17:14:19,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742009_1185 (size=30955) 2024-11-19T17:14:19,440 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=104, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/b862fad5d9954e35a74e2cd634ec5035 2024-11-19T17:14:19,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/a3c97a7486ce4117af6b1f6b2580f379 is 50, key is test_row_0/B:col10/1732036458464/Put/seqid=0 2024-11-19T17:14:19,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742010_1186 (size=12001) 2024-11-19T17:14:19,481 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/a3c97a7486ce4117af6b1f6b2580f379 2024-11-19T17:14:19,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/b77d7a72cb7b4b61a593d31ae8b9558e is 50, key is test_row_0/C:col10/1732036458464/Put/seqid=0 2024-11-19T17:14:19,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742011_1187 (size=12001) 2024-11-19T17:14:19,508 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/b77d7a72cb7b4b61a593d31ae8b9558e 2024-11-19T17:14:19,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/b862fad5d9954e35a74e2cd634ec5035 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/b862fad5d9954e35a74e2cd634ec5035 2024-11-19T17:14:19,524 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/b862fad5d9954e35a74e2cd634ec5035, entries=150, sequenceid=104, filesize=30.2 K 2024-11-19T17:14:19,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/a3c97a7486ce4117af6b1f6b2580f379 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/a3c97a7486ce4117af6b1f6b2580f379 2024-11-19T17:14:19,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,534 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/a3c97a7486ce4117af6b1f6b2580f379, entries=150, sequenceid=104, filesize=11.7 K 2024-11-19T17:14:19,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/b77d7a72cb7b4b61a593d31ae8b9558e as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/b77d7a72cb7b4b61a593d31ae8b9558e 2024-11-19T17:14:19,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,541 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/b77d7a72cb7b4b61a593d31ae8b9558e, entries=150, sequenceid=104, filesize=11.7 K 2024-11-19T17:14:19,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,542 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 5d916b1a7715625884544afede338b7a in 148ms, sequenceid=104, compaction requested=true 2024-11-19T17:14:19,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:19,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:19,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-19T17:14:19,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-19T17:14:19,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-19T17:14:19,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0770 sec 2024-11-19T17:14:19,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.0840 sec 2024-11-19T17:14:19,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T17:14:19,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,605 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:14:19,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,605 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:14:19,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-19T17:14:19,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T17:14:19,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,609 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:14:19,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,610 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:14:19,610 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:14:19,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:19,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d916b1a7715625884544afede338b7a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T17:14:19,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=A 2024-11-19T17:14:19,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:19,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=B 2024-11-19T17:14:19,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:19,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=C 2024-11-19T17:14:19,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:19,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119006b023525aa4052b4472a3abb55aa93_5d916b1a7715625884544afede338b7a is 50, key is test_row_0/A:col10/1732036459677/Put/seqid=0 2024-11-19T17:14:19,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742012_1188 (size=21918) 2024-11-19T17:14:19,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T17:14:19,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,715 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,723 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119006b023525aa4052b4472a3abb55aa93_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119006b023525aa4052b4472a3abb55aa93_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:19,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,724 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/e1de6532dbe44dbcaffa90b4c68abe39, store: [table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:19,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/e1de6532dbe44dbcaffa90b4c68abe39 is 175, key is test_row_0/A:col10/1732036459677/Put/seqid=0 2024-11-19T17:14:19,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:19,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036459717, max timestamp=1732036459735Num uniques 859; 2024-11-19T17:14:19,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:14:19,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036459717, max timestamp=1732036459735Num uniques 859; 2024-11-19T17:14:19,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:14:19,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036459717, max timestamp=1732036459734Num uniques 859; 2024-11-19T17:14:19,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:14:19,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742013_1189 (size=65323) 2024-11-19T17:14:19,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T17:14:19,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:19,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:19,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:19,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:19,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:19,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:19,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:19,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036519798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:19,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:19,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:19,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:19,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732036459783, max timestamp=1732036459806Num uniques 1018; 2024-11-19T17:14:19,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-19T17:14:19,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732036459783, max timestamp=1732036459806Num uniques 1018; 2024-11-19T17:14:19,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-19T17:14:19,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732036459783, max timestamp=1732036459806Num uniques 1018; 2024-11-19T17:14:19,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-19T17:14:19,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:19,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036519805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:19,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:19,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:19,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:19,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:19,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:19,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036519844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:19,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036519845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:19,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:19,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:19,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:19,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:19,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:19,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:19,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:19,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036519853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:19,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:19,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:19,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:19,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T17:14:19,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:19,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:19,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:19,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:19,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:19,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:19,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:19,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T17:14:19,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036519913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:19,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:19,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:19,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:19,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:19,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036519924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:19,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:19,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:19,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:19,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:19,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036519964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:19,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:19,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036519964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:19,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:19,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036519964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:19,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:19,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:19,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:19,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:19,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:19,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:19,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:19,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:19,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:20,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T17:14:20,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:20,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:20,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:20,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:20,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:20,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:20,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:20,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036520134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:20,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:20,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:20,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:20,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036520134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:20,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:20,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:20,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:20,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:20,157 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=114, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/e1de6532dbe44dbcaffa90b4c68abe39 2024-11-19T17:14:20,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/d08250d2e2e74f0cbcfba45d45bc25d0 is 50, key is test_row_0/B:col10/1732036459677/Put/seqid=0 2024-11-19T17:14:20,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742014_1190 (size=12001) 2024-11-19T17:14:20,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:20,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036520174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:20,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:20,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:20,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:20,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:20,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036520174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:20,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:20,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:20,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:20,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:20,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036520174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:20,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:20,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:20,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:20,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T17:14:20,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:20,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:20,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:20,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:20,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:20,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T17:14:20,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T17:14:20,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:20,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:20,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:20,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:20,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:20,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036520443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:20,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:20,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:20,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:20,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036520444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:20,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:20,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:20,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:20,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:20,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036520484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:20,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:20,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:20,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:20,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036520484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:20,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:20,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036520484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:20,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:20,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:20,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:20,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:20,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:20,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:20,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T17:14:20,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:20,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:20,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:20,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:20,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:20,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:20,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/d08250d2e2e74f0cbcfba45d45bc25d0 2024-11-19T17:14:20,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/c303399ac8f34915ad70e60d7ef4e6cc is 50, key is test_row_0/C:col10/1732036459677/Put/seqid=0 2024-11-19T17:14:20,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742015_1191 (size=12001) 2024-11-19T17:14:20,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/c303399ac8f34915ad70e60d7ef4e6cc 2024-11-19T17:14:20,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/e1de6532dbe44dbcaffa90b4c68abe39 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/e1de6532dbe44dbcaffa90b4c68abe39 2024-11-19T17:14:20,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/e1de6532dbe44dbcaffa90b4c68abe39, entries=350, sequenceid=114, filesize=63.8 K 2024-11-19T17:14:20,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/d08250d2e2e74f0cbcfba45d45bc25d0 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/d08250d2e2e74f0cbcfba45d45bc25d0 2024-11-19T17:14:20,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/d08250d2e2e74f0cbcfba45d45bc25d0, entries=150, sequenceid=114, filesize=11.7 K 2024-11-19T17:14:20,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/c303399ac8f34915ad70e60d7ef4e6cc as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/c303399ac8f34915ad70e60d7ef4e6cc 2024-11-19T17:14:20,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/c303399ac8f34915ad70e60d7ef4e6cc, entries=150, sequenceid=114, filesize=11.7 K 2024-11-19T17:14:20,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 5d916b1a7715625884544afede338b7a in 951ms, sequenceid=114, compaction requested=true 2024-11-19T17:14:20,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:20,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d916b1a7715625884544afede338b7a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:14:20,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:20,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d916b1a7715625884544afede338b7a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:14:20,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:20,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d916b1a7715625884544afede338b7a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:14:20,633 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:14:20,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:20,633 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:14:20,635 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 158291 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:14:20,635 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:14:20,635 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 5d916b1a7715625884544afede338b7a/A is initiating minor compaction (all files) 2024-11-19T17:14:20,635 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 5d916b1a7715625884544afede338b7a/B is initiating minor compaction (all files) 2024-11-19T17:14:20,635 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d916b1a7715625884544afede338b7a/A in TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:20,635 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d916b1a7715625884544afede338b7a/B in TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:20,635 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/ec74e8e9e56a47ed8e736ed046620633, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/5447bf1cd9c04c9f9ac82074364bf762, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/b862fad5d9954e35a74e2cd634ec5035, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/e1de6532dbe44dbcaffa90b4c68abe39] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp, totalSize=154.6 K 2024-11-19T17:14:20,635 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:20,635 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/b599280704774080bf46549ff45c5889, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/0fc07bf6a56a46658c25d7e8e42c18a7, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/a3c97a7486ce4117af6b1f6b2580f379, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/d08250d2e2e74f0cbcfba45d45bc25d0] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp, totalSize=47.0 K 2024-11-19T17:14:20,635 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. files: [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/ec74e8e9e56a47ed8e736ed046620633, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/5447bf1cd9c04c9f9ac82074364bf762, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/b862fad5d9954e35a74e2cd634ec5035, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/e1de6532dbe44dbcaffa90b4c68abe39] 2024-11-19T17:14:20,636 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec74e8e9e56a47ed8e736ed046620633, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732036457098 2024-11-19T17:14:20,636 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting b599280704774080bf46549ff45c5889, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732036457098 2024-11-19T17:14:20,636 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5447bf1cd9c04c9f9ac82074364bf762, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732036457791 2024-11-19T17:14:20,636 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fc07bf6a56a46658c25d7e8e42c18a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732036457791 2024-11-19T17:14:20,636 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting b862fad5d9954e35a74e2cd634ec5035, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732036458449 2024-11-19T17:14:20,637 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting a3c97a7486ce4117af6b1f6b2580f379, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732036458449 2024-11-19T17:14:20,637 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1de6532dbe44dbcaffa90b4c68abe39, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732036459638 2024-11-19T17:14:20,637 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting d08250d2e2e74f0cbcfba45d45bc25d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732036459677 2024-11-19T17:14:20,651 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d916b1a7715625884544afede338b7a#B#compaction#169 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:20,652 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/20589b3e89654314a85dafdc8d619236 is 50, key is test_row_0/B:col10/1732036459677/Put/seqid=0 2024-11-19T17:14:20,661 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:20,672 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119d745bef94c2e4cb2ab25ac67dde1037c_5d916b1a7715625884544afede338b7a store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:20,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742016_1192 (size=12241) 2024-11-19T17:14:20,678 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119d745bef94c2e4cb2ab25ac67dde1037c_5d916b1a7715625884544afede338b7a, store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:20,678 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d745bef94c2e4cb2ab25ac67dde1037c_5d916b1a7715625884544afede338b7a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:20,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T17:14:20,685 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/20589b3e89654314a85dafdc8d619236 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/20589b3e89654314a85dafdc8d619236 2024-11-19T17:14:20,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:20,686 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 5d916b1a7715625884544afede338b7a 3/3 column families, dataSize=140.89 KB heapSize=390 KB 2024-11-19T17:14:20,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=A 2024-11-19T17:14:20,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:20,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=B 2024-11-19T17:14:20,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:20,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=C 2024-11-19T17:14:20,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:20,694 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d916b1a7715625884544afede338b7a/B of 5d916b1a7715625884544afede338b7a into 20589b3e89654314a85dafdc8d619236(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:20,695 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:20,695 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., storeName=5d916b1a7715625884544afede338b7a/B, priority=12, startTime=1732036460633; duration=0sec 2024-11-19T17:14:20,695 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:20,695 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d916b1a7715625884544afede338b7a:B 2024-11-19T17:14:20,695 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:14:20,697 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:14:20,697 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 5d916b1a7715625884544afede338b7a/C is initiating minor compaction (all files) 2024-11-19T17:14:20,697 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d916b1a7715625884544afede338b7a/C in TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:20,697 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/923dea8eb3e348e59cbd56a21d29260f, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/ec84485fab9d4f26ac59e9bcc869427a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/b77d7a72cb7b4b61a593d31ae8b9558e, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/c303399ac8f34915ad70e60d7ef4e6cc] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp, totalSize=47.0 K 2024-11-19T17:14:20,698 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 923dea8eb3e348e59cbd56a21d29260f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732036457098 2024-11-19T17:14:20,698 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting ec84485fab9d4f26ac59e9bcc869427a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732036457791 2024-11-19T17:14:20,700 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting b77d7a72cb7b4b61a593d31ae8b9558e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732036458449 2024-11-19T17:14:20,700 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting c303399ac8f34915ad70e60d7ef4e6cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732036459677 2024-11-19T17:14:20,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111993b36dc95abe4e7c9a19cf5b8a582aa2_5d916b1a7715625884544afede338b7a is 50, key is test_row_0/A:col10/1732036459834/Put/seqid=0 2024-11-19T17:14:20,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742017_1193 (size=4469) 2024-11-19T17:14:20,717 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d916b1a7715625884544afede338b7a#A#compaction#170 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:20,717 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/4185312748934eeab86c30030e714504 is 175, key is test_row_0/A:col10/1732036459677/Put/seqid=0 2024-11-19T17:14:20,737 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d916b1a7715625884544afede338b7a#C#compaction#172 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:20,739 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/9b12eb42d87544bd8f77b1bfecac36c3 is 50, key is test_row_0/C:col10/1732036459677/Put/seqid=0 2024-11-19T17:14:20,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T17:14:20,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742018_1194 (size=12304) 2024-11-19T17:14:20,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,758 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111993b36dc95abe4e7c9a19cf5b8a582aa2_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111993b36dc95abe4e7c9a19cf5b8a582aa2_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:20,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/6393eaf22bdb459b9eda4e34caf23f35, store: [table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:20,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/6393eaf22bdb459b9eda4e34caf23f35 is 175, key is test_row_0/A:col10/1732036459834/Put/seqid=0 2024-11-19T17:14:20,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742019_1195 (size=31195) 2024-11-19T17:14:20,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742021_1197 (size=31105) 2024-11-19T17:14:20,788 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=166, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/6393eaf22bdb459b9eda4e34caf23f35 2024-11-19T17:14:20,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742020_1196 (size=12241) 2024-11-19T17:14:20,792 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/4185312748934eeab86c30030e714504 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/4185312748934eeab86c30030e714504 2024-11-19T17:14:20,800 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/9b12eb42d87544bd8f77b1bfecac36c3 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/9b12eb42d87544bd8f77b1bfecac36c3 2024-11-19T17:14:20,802 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d916b1a7715625884544afede338b7a/A of 5d916b1a7715625884544afede338b7a into 4185312748934eeab86c30030e714504(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:20,802 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:20,802 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., storeName=5d916b1a7715625884544afede338b7a/A, priority=12, startTime=1732036460633; duration=0sec 2024-11-19T17:14:20,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/c42291dbb2bc44ec850bc3e333b44858 is 50, key is test_row_0/B:col10/1732036459834/Put/seqid=0 2024-11-19T17:14:20,803 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:20,803 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d916b1a7715625884544afede338b7a:A 2024-11-19T17:14:20,809 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d916b1a7715625884544afede338b7a/C of 5d916b1a7715625884544afede338b7a into 9b12eb42d87544bd8f77b1bfecac36c3(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:20,809 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:20,809 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., storeName=5d916b1a7715625884544afede338b7a/C, priority=12, startTime=1732036460633; duration=0sec 2024-11-19T17:14:20,810 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:20,810 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d916b1a7715625884544afede338b7a:C 2024-11-19T17:14:20,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742022_1198 (size=12151) 2024-11-19T17:14:20,833 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/c42291dbb2bc44ec850bc3e333b44858 2024-11-19T17:14:20,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/ec872f2e4fd84e1590fc0a285e59e71b is 50, key is test_row_0/C:col10/1732036459834/Put/seqid=0 2024-11-19T17:14:20,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742023_1199 (size=12151) 2024-11-19T17:14:20,884 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/ec872f2e4fd84e1590fc0a285e59e71b 2024-11-19T17:14:20,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/6393eaf22bdb459b9eda4e34caf23f35 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/6393eaf22bdb459b9eda4e34caf23f35 2024-11-19T17:14:20,897 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/6393eaf22bdb459b9eda4e34caf23f35, entries=150, sequenceid=166, filesize=30.4 K 2024-11-19T17:14:20,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/c42291dbb2bc44ec850bc3e333b44858 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/c42291dbb2bc44ec850bc3e333b44858 2024-11-19T17:14:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,908 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/c42291dbb2bc44ec850bc3e333b44858, entries=150, sequenceid=166, filesize=11.9 K 2024-11-19T17:14:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/ec872f2e4fd84e1590fc0a285e59e71b as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/ec872f2e4fd84e1590fc0a285e59e71b 2024-11-19T17:14:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,916 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/ec872f2e4fd84e1590fc0a285e59e71b, entries=150, sequenceid=166, filesize=11.9 K 2024-11-19T17:14:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,918 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~389.95 KB/399312, currentSize=0 B/0 for 5d916b1a7715625884544afede338b7a in 232ms, sequenceid=166, compaction requested=false 2024-11-19T17:14:20,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:20,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-19T17:14:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-19T17:14:20,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,926 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-19T17:14:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,926 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3130 sec 2024-11-19T17:14:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.3210 sec 2024-11-19T17:14:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:20,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,096 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d916b1a7715625884544afede338b7a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T17:14:21,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:21,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=A 2024-11-19T17:14:21,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:21,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=B 2024-11-19T17:14:21,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:21,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=C 2024-11-19T17:14:21,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:21,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c22db7fd04704d529eefd3d0e3558c36_5d916b1a7715625884544afede338b7a is 50, key is test_row_0/A:col10/1732036461088/Put/seqid=0 2024-11-19T17:14:21,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:21,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036521145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:21,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:21,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:21,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036521146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:21,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:21,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036521147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:21,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:21,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:21,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:21,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036521149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:21,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:21,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036521151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:21,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:21,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:21,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742025_1201 (size=27248) 2024-11-19T17:14:21,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:21,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036521264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:21,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:21,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036521264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:21,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:21,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:21,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036521264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:21,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:21,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:21,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036521264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:21,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036521264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:21,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:21,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:21,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:21,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:21,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:21,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036521474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:21,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:21,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036521474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:21,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:21,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036521474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:21,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:21,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:21,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036521484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036521484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:21,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:21,586 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:21,592 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c22db7fd04704d529eefd3d0e3558c36_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c22db7fd04704d529eefd3d0e3558c36_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:21,594 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/a9b82d2de3e14e208789283566038e2d, store: [table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:21,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/a9b82d2de3e14e208789283566038e2d is 175, key is test_row_0/A:col10/1732036461088/Put/seqid=0 2024-11-19T17:14:21,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742024_1200 (size=83035) 2024-11-19T17:14:21,599 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=181, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/a9b82d2de3e14e208789283566038e2d 2024-11-19T17:14:21,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/cef3e4d46f18423f84f151e21e867bca is 50, key is test_row_0/B:col10/1732036461088/Put/seqid=0 2024-11-19T17:14:21,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742026_1202 (size=12151) 2024-11-19T17:14:21,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/cef3e4d46f18423f84f151e21e867bca 2024-11-19T17:14:21,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/f2a3671ca1b34cecbe848b2228ee3414 is 50, key is test_row_0/C:col10/1732036461088/Put/seqid=0 2024-11-19T17:14:21,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742027_1203 (size=12151) 2024-11-19T17:14:21,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T17:14:21,754 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:14:21,755 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:14:21,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-19T17:14:21,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T17:14:21,758 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:14:21,759 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:14:21,759 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:14:21,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036521784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:21,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:21,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036521794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:21,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036521794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:21,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036521794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:21,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:21,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:21,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036521794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:21,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:21,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:21,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T17:14:21,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T17:14:21,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:21,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:21,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:21,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:21,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:21,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:22,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T17:14:22,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:22,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:22,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:22,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:22,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T17:14:22,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/f2a3671ca1b34cecbe848b2228ee3414 2024-11-19T17:14:22,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/a9b82d2de3e14e208789283566038e2d as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/a9b82d2de3e14e208789283566038e2d 2024-11-19T17:14:22,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/a9b82d2de3e14e208789283566038e2d, entries=450, sequenceid=181, filesize=81.1 K 2024-11-19T17:14:22,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/cef3e4d46f18423f84f151e21e867bca as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/cef3e4d46f18423f84f151e21e867bca 2024-11-19T17:14:22,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/cef3e4d46f18423f84f151e21e867bca, entries=150, sequenceid=181, filesize=11.9 K 2024-11-19T17:14:22,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/f2a3671ca1b34cecbe848b2228ee3414 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/f2a3671ca1b34cecbe848b2228ee3414 2024-11-19T17:14:22,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/f2a3671ca1b34cecbe848b2228ee3414, entries=150, sequenceid=181, filesize=11.9 K 2024-11-19T17:14:22,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5d916b1a7715625884544afede338b7a in 1077ms, sequenceid=181, compaction requested=true 2024-11-19T17:14:22,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:22,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d916b1a7715625884544afede338b7a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:14:22,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:22,173 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:22,173 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:22,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d916b1a7715625884544afede338b7a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:14:22,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:22,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d916b1a7715625884544afede338b7a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:14:22,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:22,176 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 145335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:22,176 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:22,176 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 5d916b1a7715625884544afede338b7a/B is initiating minor compaction (all files) 2024-11-19T17:14:22,176 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 5d916b1a7715625884544afede338b7a/A is initiating minor compaction (all files) 2024-11-19T17:14:22,176 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d916b1a7715625884544afede338b7a/B in TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:22,176 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d916b1a7715625884544afede338b7a/A in TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:22,176 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/4185312748934eeab86c30030e714504, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/6393eaf22bdb459b9eda4e34caf23f35, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/a9b82d2de3e14e208789283566038e2d] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp, totalSize=141.9 K 2024-11-19T17:14:22,176 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/20589b3e89654314a85dafdc8d619236, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/c42291dbb2bc44ec850bc3e333b44858, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/cef3e4d46f18423f84f151e21e867bca] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp, totalSize=35.7 K 2024-11-19T17:14:22,177 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:22,177 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. files: [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/4185312748934eeab86c30030e714504, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/6393eaf22bdb459b9eda4e34caf23f35, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/a9b82d2de3e14e208789283566038e2d] 2024-11-19T17:14:22,181 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 20589b3e89654314a85dafdc8d619236, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732036459677 2024-11-19T17:14:22,181 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4185312748934eeab86c30030e714504, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732036459677 2024-11-19T17:14:22,181 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting c42291dbb2bc44ec850bc3e333b44858, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732036459834 2024-11-19T17:14:22,182 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6393eaf22bdb459b9eda4e34caf23f35, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732036459834 2024-11-19T17:14:22,182 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9b82d2de3e14e208789283566038e2d, keycount=450, bloomtype=ROW, size=81.1 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732036461081 2024-11-19T17:14:22,182 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting cef3e4d46f18423f84f151e21e867bca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732036461086 2024-11-19T17:14:22,202 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d916b1a7715625884544afede338b7a#B#compaction#178 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:22,203 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/beb994ac92c5473db2e0d995ece88e11 is 50, key is test_row_0/B:col10/1732036461088/Put/seqid=0 2024-11-19T17:14:22,204 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:22,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742028_1204 (size=12493) 2024-11-19T17:14:22,219 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119746291fffacc48d2be657e887b2854f7_5d916b1a7715625884544afede338b7a store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:22,223 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119746291fffacc48d2be657e887b2854f7_5d916b1a7715625884544afede338b7a, store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:22,223 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119746291fffacc48d2be657e887b2854f7_5d916b1a7715625884544afede338b7a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:22,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T17:14:22,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:22,226 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 5d916b1a7715625884544afede338b7a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T17:14:22,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=A 2024-11-19T17:14:22,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:22,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=B 2024-11-19T17:14:22,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:22,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=C 2024-11-19T17:14:22,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:22,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742029_1205 (size=4469) 2024-11-19T17:14:22,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c0fb4d55af6749e09273892c8e274363_5d916b1a7715625884544afede338b7a is 50, key is test_row_0/A:col10/1732036461149/Put/seqid=0 2024-11-19T17:14:22,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742030_1206 (size=12304) 2024-11-19T17:14:22,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:22,307 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c0fb4d55af6749e09273892c8e274363_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c0fb4d55af6749e09273892c8e274363_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:22,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/fc4c03a222ea434991a1ab703c95f7fb, store: [table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:22,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/fc4c03a222ea434991a1ab703c95f7fb is 175, key is test_row_0/A:col10/1732036461149/Put/seqid=0 2024-11-19T17:14:22,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:22,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036522317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:22,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036522320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:22,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036522322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:22,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:22,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:22,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036522323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:22,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036522325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:22,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:22,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742031_1207 (size=31105) 2024-11-19T17:14:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T17:14:22,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036522434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:22,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:22,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036522444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:22,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:22,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036522444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:22,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:22,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036522444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:22,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:22,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036522444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:22,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:22,619 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/beb994ac92c5473db2e0d995ece88e11 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/beb994ac92c5473db2e0d995ece88e11 2024-11-19T17:14:22,626 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d916b1a7715625884544afede338b7a/B of 5d916b1a7715625884544afede338b7a into beb994ac92c5473db2e0d995ece88e11(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:22,626 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:22,626 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., storeName=5d916b1a7715625884544afede338b7a/B, priority=13, startTime=1732036462173; duration=0sec 2024-11-19T17:14:22,626 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:22,626 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d916b1a7715625884544afede338b7a:B 2024-11-19T17:14:22,626 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:22,627 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:22,627 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 5d916b1a7715625884544afede338b7a/C is initiating minor compaction (all files) 2024-11-19T17:14:22,627 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d916b1a7715625884544afede338b7a/C in TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:22,627 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/9b12eb42d87544bd8f77b1bfecac36c3, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/ec872f2e4fd84e1590fc0a285e59e71b, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/f2a3671ca1b34cecbe848b2228ee3414] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp, totalSize=35.7 K 2024-11-19T17:14:22,628 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b12eb42d87544bd8f77b1bfecac36c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732036459677 2024-11-19T17:14:22,628 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting ec872f2e4fd84e1590fc0a285e59e71b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732036459834 2024-11-19T17:14:22,629 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting f2a3671ca1b34cecbe848b2228ee3414, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732036461086 2024-11-19T17:14:22,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036522644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:22,650 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d916b1a7715625884544afede338b7a#A#compaction#179 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:22,651 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d916b1a7715625884544afede338b7a#C#compaction#181 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:22,651 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/ef45c671b8c54f69b17ef2b613b722db is 175, key is test_row_0/A:col10/1732036461088/Put/seqid=0 2024-11-19T17:14:22,651 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/0dc178fada1240e8b0e0b0bbde20a8da is 50, key is test_row_0/C:col10/1732036461088/Put/seqid=0 2024-11-19T17:14:22,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742032_1208 (size=31447) 2024-11-19T17:14:22,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036522654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:22,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:22,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036522654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:22,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:22,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:22,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036522664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:22,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:22,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036522664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:22,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:22,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:22,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742033_1209 (size=12493) 2024-11-19T17:14:22,756 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=201, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/fc4c03a222ea434991a1ab703c95f7fb 2024-11-19T17:14:22,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/68ee75dbbf984b468902a4599e49071a is 50, key is test_row_0/B:col10/1732036461149/Put/seqid=0 2024-11-19T17:14:22,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742034_1210 (size=12151) 2024-11-19T17:14:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T17:14:22,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036522955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:22,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:22,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036522964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:22,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:22,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036522973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:22,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036522973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:22,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:22,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036522973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:22,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:22,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:23,066 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/ef45c671b8c54f69b17ef2b613b722db as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/ef45c671b8c54f69b17ef2b613b722db 2024-11-19T17:14:23,073 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d916b1a7715625884544afede338b7a/A of 5d916b1a7715625884544afede338b7a into ef45c671b8c54f69b17ef2b613b722db(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:23,073 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:23,073 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., storeName=5d916b1a7715625884544afede338b7a/A, priority=13, startTime=1732036462173; duration=0sec 2024-11-19T17:14:23,073 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:23,073 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d916b1a7715625884544afede338b7a:A 2024-11-19T17:14:23,092 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/0dc178fada1240e8b0e0b0bbde20a8da as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/0dc178fada1240e8b0e0b0bbde20a8da 2024-11-19T17:14:23,099 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d916b1a7715625884544afede338b7a/C of 5d916b1a7715625884544afede338b7a into 0dc178fada1240e8b0e0b0bbde20a8da(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:23,099 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:23,099 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., storeName=5d916b1a7715625884544afede338b7a/C, priority=13, startTime=1732036462174; duration=0sec 2024-11-19T17:14:23,100 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:23,100 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d916b1a7715625884544afede338b7a:C 2024-11-19T17:14:23,194 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/68ee75dbbf984b468902a4599e49071a 2024-11-19T17:14:23,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/b710f93f0b094d829b5702ec9e4045a5 is 50, key is test_row_0/C:col10/1732036461149/Put/seqid=0 2024-11-19T17:14:23,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742035_1211 (size=12151) 2024-11-19T17:14:23,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036523464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:23,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:23,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:23,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:23,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036523473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:23,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:23,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036523484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:23,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:23,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036523484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:23,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:23,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:23,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:23,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:23,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:23,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:23,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:23,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036523486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:23,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:23,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:23,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:23,621 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/b710f93f0b094d829b5702ec9e4045a5 2024-11-19T17:14:23,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/fc4c03a222ea434991a1ab703c95f7fb as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/fc4c03a222ea434991a1ab703c95f7fb 2024-11-19T17:14:23,638 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/fc4c03a222ea434991a1ab703c95f7fb, entries=150, sequenceid=201, filesize=30.4 K 2024-11-19T17:14:23,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/68ee75dbbf984b468902a4599e49071a as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/68ee75dbbf984b468902a4599e49071a 2024-11-19T17:14:23,645 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/68ee75dbbf984b468902a4599e49071a, entries=150, sequenceid=201, filesize=11.9 K 2024-11-19T17:14:23,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/b710f93f0b094d829b5702ec9e4045a5 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/b710f93f0b094d829b5702ec9e4045a5 2024-11-19T17:14:23,654 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/b710f93f0b094d829b5702ec9e4045a5, entries=150, sequenceid=201, filesize=11.9 K 2024-11-19T17:14:23,656 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 5d916b1a7715625884544afede338b7a in 1429ms, sequenceid=201, compaction requested=false 2024-11-19T17:14:23,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:23,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:23,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-19T17:14:23,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-19T17:14:23,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-19T17:14:23,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8990 sec 2024-11-19T17:14:23,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.9070 sec 2024-11-19T17:14:23,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T17:14:23,904 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:14:23,905 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:14:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-19T17:14:23,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T17:14:23,907 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:14:23,908 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:14:23,908 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:14:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T17:14:24,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T17:14:24,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:24,062 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 5d916b1a7715625884544afede338b7a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T17:14:24,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=A 2024-11-19T17:14:24,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:24,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=B 2024-11-19T17:14:24,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:24,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=C 2024-11-19T17:14:24,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:24,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111992a1d804dc8d4c1e849d722574bb0e4f_5d916b1a7715625884544afede338b7a is 50, key is test_row_0/A:col10/1732036462323/Put/seqid=0 2024-11-19T17:14:24,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742036_1212 (size=12304) 2024-11-19T17:14:24,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:24,091 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111992a1d804dc8d4c1e849d722574bb0e4f_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111992a1d804dc8d4c1e849d722574bb0e4f_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:24,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/de841b53fbcc4059a4c71ccb4c05c15c, store: [table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:24,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/de841b53fbcc4059a4c71ccb4c05c15c is 175, key is test_row_0/A:col10/1732036462323/Put/seqid=0 2024-11-19T17:14:24,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742037_1213 (size=31105) 2024-11-19T17:14:24,098 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/de841b53fbcc4059a4c71ccb4c05c15c 2024-11-19T17:14:24,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/08b531d2997f42fd8b7e421eda7e7457 is 50, key is test_row_0/B:col10/1732036462323/Put/seqid=0 2024-11-19T17:14:24,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742038_1214 (size=12151) 2024-11-19T17:14:24,123 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/08b531d2997f42fd8b7e421eda7e7457 2024-11-19T17:14:24,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/5d0c8452d3004ce6a14b5565f5be2386 is 50, key is test_row_0/C:col10/1732036462323/Put/seqid=0 2024-11-19T17:14:24,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742039_1215 (size=12151) 2024-11-19T17:14:24,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T17:14:24,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:24,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:24,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:24,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036524518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:24,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:24,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036524521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:24,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:24,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036524521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:24,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:24,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:24,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036524523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:24,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:24,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:24,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036524524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:24,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T17:14:24,545 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/5d0c8452d3004ce6a14b5565f5be2386 2024-11-19T17:14:24,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/de841b53fbcc4059a4c71ccb4c05c15c as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/de841b53fbcc4059a4c71ccb4c05c15c 2024-11-19T17:14:24,561 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/de841b53fbcc4059a4c71ccb4c05c15c, entries=150, sequenceid=216, filesize=30.4 K 2024-11-19T17:14:24,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/08b531d2997f42fd8b7e421eda7e7457 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/08b531d2997f42fd8b7e421eda7e7457 2024-11-19T17:14:24,569 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/08b531d2997f42fd8b7e421eda7e7457, entries=150, sequenceid=216, filesize=11.9 K 2024-11-19T17:14:24,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/5d0c8452d3004ce6a14b5565f5be2386 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/5d0c8452d3004ce6a14b5565f5be2386 2024-11-19T17:14:24,575 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/5d0c8452d3004ce6a14b5565f5be2386, entries=150, sequenceid=216, filesize=11.9 K 2024-11-19T17:14:24,576 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5d916b1a7715625884544afede338b7a in 514ms, sequenceid=216, compaction requested=true 2024-11-19T17:14:24,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:24,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:24,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-19T17:14:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-19T17:14:24,581 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-19T17:14:24,581 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 670 msec 2024-11-19T17:14:24,583 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 677 msec 2024-11-19T17:14:24,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036464517, max timestamp=1732036464634Num uniques 859; 2024-11-19T17:14:24,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:14:24,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036464517, max timestamp=1732036464634Num uniques 859; 2024-11-19T17:14:24,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:14:24,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036464517, max timestamp=1732036464634Num uniques 859; 2024-11-19T17:14:24,640 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:14:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:24,644 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d916b1a7715625884544afede338b7a 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-19T17:14:24,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=A 2024-11-19T17:14:24,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:24,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=B 2024-11-19T17:14:24,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:24,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=C 2024-11-19T17:14:24,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:24,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119824688cd745b4014904c5652a1564a66_5d916b1a7715625884544afede338b7a is 50, key is test_row_0/A:col10/1732036464637/Put/seqid=0 2024-11-19T17:14:24,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742040_1216 (size=14794) 2024-11-19T17:14:24,664 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:24,669 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119824688cd745b4014904c5652a1564a66_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119824688cd745b4014904c5652a1564a66_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:24,670 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/24b583e3f5fc496e9045baa84202763d, store: [table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:24,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/24b583e3f5fc496e9045baa84202763d is 175, key is test_row_0/A:col10/1732036464637/Put/seqid=0 2024-11-19T17:14:24,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036464662, max timestamp=1732036464676Num uniques 859; 2024-11-19T17:14:24,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036464662, max timestamp=1732036464676Num uniques 859; 2024-11-19T17:14:24,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:14:24,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:14:24,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036464662, max timestamp=1732036464676Num uniques 859; 2024-11-19T17:14:24,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:14:24,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742041_1217 (size=39749) 2024-11-19T17:14:24,687 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=241, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/24b583e3f5fc496e9045baa84202763d 2024-11-19T17:14:24,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/4ad2ae9bb2af43efa9c49fb508283bae is 50, key is test_row_0/B:col10/1732036464637/Put/seqid=0 2024-11-19T17:14:24,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036524708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:24,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:24,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742042_1218 (size=12151) 2024-11-19T17:14:24,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/4ad2ae9bb2af43efa9c49fb508283bae 2024-11-19T17:14:24,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036524709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:24,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:24,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036524710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:24,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:24,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036524711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:24,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:24,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036524712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:24,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:24,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:24,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/a2767f2e1ec9431d9ac4fe2a0ce578fe is 50, key is test_row_0/C:col10/1732036464637/Put/seqid=0 2024-11-19T17:14:24,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742043_1219 (size=12151) 2024-11-19T17:14:24,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/a2767f2e1ec9431d9ac4fe2a0ce578fe 2024-11-19T17:14:24,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/24b583e3f5fc496e9045baa84202763d as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/24b583e3f5fc496e9045baa84202763d 2024-11-19T17:14:24,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/24b583e3f5fc496e9045baa84202763d, entries=200, sequenceid=241, filesize=38.8 K 2024-11-19T17:14:24,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/4ad2ae9bb2af43efa9c49fb508283bae as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/4ad2ae9bb2af43efa9c49fb508283bae 2024-11-19T17:14:24,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/4ad2ae9bb2af43efa9c49fb508283bae, entries=150, sequenceid=241, filesize=11.9 K 2024-11-19T17:14:24,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/a2767f2e1ec9431d9ac4fe2a0ce578fe as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/a2767f2e1ec9431d9ac4fe2a0ce578fe 2024-11-19T17:14:24,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/a2767f2e1ec9431d9ac4fe2a0ce578fe, entries=150, sequenceid=241, filesize=11.9 K 2024-11-19T17:14:24,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=134.18 KB/137400 for 5d916b1a7715625884544afede338b7a in 128ms, sequenceid=241, compaction requested=true 2024-11-19T17:14:24,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:24,771 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:14:24,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d916b1a7715625884544afede338b7a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:14:24,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:24,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d916b1a7715625884544afede338b7a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:14:24,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:24,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d916b1a7715625884544afede338b7a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:14:24,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T17:14:24,772 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:14:24,774 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:14:24,774 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 5d916b1a7715625884544afede338b7a/B is initiating minor compaction (all files) 2024-11-19T17:14:24,774 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d916b1a7715625884544afede338b7a/B in TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:24,774 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/beb994ac92c5473db2e0d995ece88e11, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/68ee75dbbf984b468902a4599e49071a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/08b531d2997f42fd8b7e421eda7e7457, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/4ad2ae9bb2af43efa9c49fb508283bae] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp, totalSize=47.8 K 2024-11-19T17:14:24,775 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133406 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:14:24,775 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 5d916b1a7715625884544afede338b7a/A is initiating minor compaction (all files) 2024-11-19T17:14:24,775 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d916b1a7715625884544afede338b7a/A in TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:24,775 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/ef45c671b8c54f69b17ef2b613b722db, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/fc4c03a222ea434991a1ab703c95f7fb, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/de841b53fbcc4059a4c71ccb4c05c15c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/24b583e3f5fc496e9045baa84202763d] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp, totalSize=130.3 K 2024-11-19T17:14:24,775 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:24,775 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. files: [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/ef45c671b8c54f69b17ef2b613b722db, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/fc4c03a222ea434991a1ab703c95f7fb, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/de841b53fbcc4059a4c71ccb4c05c15c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/24b583e3f5fc496e9045baa84202763d] 2024-11-19T17:14:24,775 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting beb994ac92c5473db2e0d995ece88e11, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732036461086 2024-11-19T17:14:24,776 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting ef45c671b8c54f69b17ef2b613b722db, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732036461086 2024-11-19T17:14:24,776 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting fc4c03a222ea434991a1ab703c95f7fb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732036461144 2024-11-19T17:14:24,776 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68ee75dbbf984b468902a4599e49071a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732036461144 2024-11-19T17:14:24,777 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08b531d2997f42fd8b7e421eda7e7457, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732036462313 2024-11-19T17:14:24,777 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting de841b53fbcc4059a4c71ccb4c05c15c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732036462313 2024-11-19T17:14:24,778 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ad2ae9bb2af43efa9c49fb508283bae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732036464637 2024-11-19T17:14:24,778 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 24b583e3f5fc496e9045baa84202763d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732036464520 2024-11-19T17:14:24,796 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:24,808 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d916b1a7715625884544afede338b7a#B#compaction#190 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:24,809 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/74d6483b2b824c53a083b27e44dedecb is 50, key is test_row_0/B:col10/1732036464637/Put/seqid=0 2024-11-19T17:14:24,809 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111903fd14ac66f54134a2ba2a8e4fb683d3_5d916b1a7715625884544afede338b7a store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:24,812 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111903fd14ac66f54134a2ba2a8e4fb683d3_5d916b1a7715625884544afede338b7a, store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:24,812 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111903fd14ac66f54134a2ba2a8e4fb683d3_5d916b1a7715625884544afede338b7a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:24,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d916b1a7715625884544afede338b7a 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-19T17:14:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:24,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=A 2024-11-19T17:14:24,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:24,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=B 2024-11-19T17:14:24,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:24,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=C 2024-11-19T17:14:24,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:24,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036524835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:24,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:24,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036524835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:24,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:24,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:24,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742044_1220 (size=12629) 2024-11-19T17:14:24,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036524844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:24,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:24,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036524847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:24,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036524846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:24,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:24,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:24,871 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/74d6483b2b824c53a083b27e44dedecb as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/74d6483b2b824c53a083b27e44dedecb 2024-11-19T17:14:24,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742045_1221 (size=4469) 2024-11-19T17:14:24,879 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d916b1a7715625884544afede338b7a#A#compaction#191 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:24,881 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/4d7bd0e29f4c4004bb2a120f0471eafb is 175, key is test_row_0/A:col10/1732036464637/Put/seqid=0 2024-11-19T17:14:24,882 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d916b1a7715625884544afede338b7a/B of 5d916b1a7715625884544afede338b7a into 74d6483b2b824c53a083b27e44dedecb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:24,882 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:24,882 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., storeName=5d916b1a7715625884544afede338b7a/B, priority=12, startTime=1732036464771; duration=0sec 2024-11-19T17:14:24,882 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:24,882 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d916b1a7715625884544afede338b7a:B 2024-11-19T17:14:24,882 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:14:24,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196ef65326e31443b7b8cd006f40e97489_5d916b1a7715625884544afede338b7a is 50, key is test_row_0/A:col10/1732036464708/Put/seqid=0 2024-11-19T17:14:24,892 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:14:24,892 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 5d916b1a7715625884544afede338b7a/C is initiating minor compaction (all files) 2024-11-19T17:14:24,892 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d916b1a7715625884544afede338b7a/C in TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:24,892 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/0dc178fada1240e8b0e0b0bbde20a8da, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/b710f93f0b094d829b5702ec9e4045a5, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/5d0c8452d3004ce6a14b5565f5be2386, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/a2767f2e1ec9431d9ac4fe2a0ce578fe] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp, totalSize=47.8 K 2024-11-19T17:14:24,893 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dc178fada1240e8b0e0b0bbde20a8da, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732036461086 2024-11-19T17:14:24,893 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting b710f93f0b094d829b5702ec9e4045a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732036461144 2024-11-19T17:14:24,894 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d0c8452d3004ce6a14b5565f5be2386, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732036462313 2024-11-19T17:14:24,894 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2767f2e1ec9431d9ac4fe2a0ce578fe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732036464637 2024-11-19T17:14:24,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742046_1222 (size=31583) 2024-11-19T17:14:24,918 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d916b1a7715625884544afede338b7a#C#compaction#193 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:24,919 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/21929e2424c74fe684ab756f47962a10 is 50, key is test_row_0/C:col10/1732036464637/Put/seqid=0 2024-11-19T17:14:24,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742047_1223 (size=12454) 2024-11-19T17:14:24,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742048_1224 (size=12629) 2024-11-19T17:14:24,928 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:24,935 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196ef65326e31443b7b8cd006f40e97489_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196ef65326e31443b7b8cd006f40e97489_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:24,937 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/e17196e1c4d941e89b4cc6fc998a328b, store: [table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:24,938 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/21929e2424c74fe684ab756f47962a10 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/21929e2424c74fe684ab756f47962a10 2024-11-19T17:14:24,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/e17196e1c4d941e89b4cc6fc998a328b is 175, key is test_row_0/A:col10/1732036464708/Put/seqid=0 2024-11-19T17:14:24,943 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d916b1a7715625884544afede338b7a/C of 5d916b1a7715625884544afede338b7a into 21929e2424c74fe684ab756f47962a10(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:24,943 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:24,943 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., storeName=5d916b1a7715625884544afede338b7a/C, priority=12, startTime=1732036464771; duration=0sec 2024-11-19T17:14:24,943 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:24,943 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d916b1a7715625884544afede338b7a:C 2024-11-19T17:14:24,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742049_1225 (size=31255) 2024-11-19T17:14:24,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036524954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:24,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036524955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:24,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:24,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:24,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036524964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:24,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:24,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036524964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:24,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:24,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:24,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036524973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:24,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:25,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T17:14:25,044 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:14:25,045 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:14:25,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-19T17:14:25,049 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:14:25,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T17:14:25,050 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:14:25,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:14:25,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T17:14:25,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:25,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036525164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:25,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:25,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:25,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036525164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:25,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:25,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:25,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036525176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:25,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036525176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:25,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036525183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:25,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T17:14:25,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:25,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:25,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:25,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:25,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:25,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:25,328 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/4d7bd0e29f4c4004bb2a120f0471eafb as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/4d7bd0e29f4c4004bb2a120f0471eafb 2024-11-19T17:14:25,336 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d916b1a7715625884544afede338b7a/A of 5d916b1a7715625884544afede338b7a into 4d7bd0e29f4c4004bb2a120f0471eafb(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:25,336 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:25,336 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., storeName=5d916b1a7715625884544afede338b7a/A, priority=12, startTime=1732036464771; duration=0sec 2024-11-19T17:14:25,336 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:25,336 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d916b1a7715625884544afede338b7a:A 2024-11-19T17:14:25,348 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/e17196e1c4d941e89b4cc6fc998a328b 2024-11-19T17:14:25,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/64e2d1bf96f047f38da96274db5624ae is 50, key is test_row_0/B:col10/1732036464708/Put/seqid=0 2024-11-19T17:14:25,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T17:14:25,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:25,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:25,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:25,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:25,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:25,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:25,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T17:14:25,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742050_1226 (size=12301) 2024-11-19T17:14:25,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/64e2d1bf96f047f38da96274db5624ae 2024-11-19T17:14:25,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/4168946b95ca46319c589f7e6fafe7b5 is 50, key is test_row_0/C:col10/1732036464708/Put/seqid=0 2024-11-19T17:14:25,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742051_1227 (size=12301) 2024-11-19T17:14:25,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/4168946b95ca46319c589f7e6fafe7b5 2024-11-19T17:14:25,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/e17196e1c4d941e89b4cc6fc998a328b as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/e17196e1c4d941e89b4cc6fc998a328b 2024-11-19T17:14:25,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/e17196e1c4d941e89b4cc6fc998a328b, entries=150, sequenceid=278, filesize=30.5 K 2024-11-19T17:14:25,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/64e2d1bf96f047f38da96274db5624ae as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/64e2d1bf96f047f38da96274db5624ae 2024-11-19T17:14:25,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/64e2d1bf96f047f38da96274db5624ae, entries=150, sequenceid=278, filesize=12.0 K 2024-11-19T17:14:25,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/4168946b95ca46319c589f7e6fafe7b5 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/4168946b95ca46319c589f7e6fafe7b5 2024-11-19T17:14:25,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/4168946b95ca46319c589f7e6fafe7b5, entries=150, sequenceid=278, filesize=12.0 K 2024-11-19T17:14:25,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,453 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 5d916b1a7715625884544afede338b7a in 625ms, sequenceid=278, compaction requested=false 2024-11-19T17:14:25,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:25,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:25,479 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d916b1a7715625884544afede338b7a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T17:14:25,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=A 2024-11-19T17:14:25,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:25,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=B 2024-11-19T17:14:25,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:25,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=C 2024-11-19T17:14:25,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:25,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192ed6b37ec4714c61b7568dc249644477_5d916b1a7715625884544afede338b7a is 50, key is test_row_0/A:col10/1732036464833/Put/seqid=0 2024-11-19T17:14:25,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T17:14:25,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:25,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:25,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:25,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:25,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:25,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:25,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732036465510, max timestamp=1732036465530Num uniques 859; 2024-11-19T17:14:25,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T17:14:25,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732036465510, max timestamp=1732036465530Num uniques 859; 2024-11-19T17:14:25,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T17:14:25,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732036465510, max timestamp=1732036465530Num uniques 859; 2024-11-19T17:14:25,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T17:14:25,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742052_1228 (size=12454) 2024-11-19T17:14:25,549 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:25,555 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192ed6b37ec4714c61b7568dc249644477_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192ed6b37ec4714c61b7568dc249644477_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:25,559 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/ed214672a3774e048dce0f4e6c8c2315, store: [table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:25,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:25,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/ed214672a3774e048dce0f4e6c8c2315 is 175, key is test_row_0/A:col10/1732036464833/Put/seqid=0 2024-11-19T17:14:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036525554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:25,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:25,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036525557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:25,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:25,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:25,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036525559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:25,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036525559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:25,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:25,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036525560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:25,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:25,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:25,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742053_1229 (size=31255) 2024-11-19T17:14:25,574 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/ed214672a3774e048dce0f4e6c8c2315 2024-11-19T17:14:25,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/946da17212fa4adb8844b9f6e72179ea is 50, key is test_row_0/B:col10/1732036464833/Put/seqid=0 2024-11-19T17:14:25,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742054_1230 (size=12301) 2024-11-19T17:14:25,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T17:14:25,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:25,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:25,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:25,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:25,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:25,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:25,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:25,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036525664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T17:14:25,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:25,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:25,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036525675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:25,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:25,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036525675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:25,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:25,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:25,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036525675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:25,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:25,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:25,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036525676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:25,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:25,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:25,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T17:14:25,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:25,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:25,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:25,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:25,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:25,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:25,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:25,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036525884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:25,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:25,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036525884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:25,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:25,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:25,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:25,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036525884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:25,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:25,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036525884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:25,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:25,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:25,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:25,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036525885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:25,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:25,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:25,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T17:14:25,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:25,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:25,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:25,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:25,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:25,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:26,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/946da17212fa4adb8844b9f6e72179ea 2024-11-19T17:14:26,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/08dfd8a435764d06b911e84d05619056 is 50, key is test_row_0/C:col10/1732036464833/Put/seqid=0 2024-11-19T17:14:26,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742055_1231 (size=12301) 2024-11-19T17:14:26,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/08dfd8a435764d06b911e84d05619056 2024-11-19T17:14:26,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/ed214672a3774e048dce0f4e6c8c2315 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/ed214672a3774e048dce0f4e6c8c2315 2024-11-19T17:14:26,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/ed214672a3774e048dce0f4e6c8c2315, entries=150, sequenceid=291, filesize=30.5 K 2024-11-19T17:14:26,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/946da17212fa4adb8844b9f6e72179ea as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/946da17212fa4adb8844b9f6e72179ea 2024-11-19T17:14:26,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/946da17212fa4adb8844b9f6e72179ea, entries=150, sequenceid=291, filesize=12.0 K 2024-11-19T17:14:26,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/08dfd8a435764d06b911e84d05619056 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/08dfd8a435764d06b911e84d05619056 2024-11-19T17:14:26,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/08dfd8a435764d06b911e84d05619056, entries=150, sequenceid=291, filesize=12.0 K 2024-11-19T17:14:26,067 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 5d916b1a7715625884544afede338b7a in 588ms, sequenceid=291, compaction requested=true 2024-11-19T17:14:26,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:26,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d916b1a7715625884544afede338b7a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:14:26,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:26,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d916b1a7715625884544afede338b7a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:14:26,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:26,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d916b1a7715625884544afede338b7a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:14:26,067 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:26,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:26,067 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:26,069 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:26,069 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:26,069 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 5d916b1a7715625884544afede338b7a/A is initiating minor compaction (all files) 2024-11-19T17:14:26,069 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 5d916b1a7715625884544afede338b7a/B is initiating minor compaction (all files) 2024-11-19T17:14:26,069 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d916b1a7715625884544afede338b7a/A in TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:26,069 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d916b1a7715625884544afede338b7a/B in TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:26,069 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/4d7bd0e29f4c4004bb2a120f0471eafb, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/e17196e1c4d941e89b4cc6fc998a328b, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/ed214672a3774e048dce0f4e6c8c2315] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp, totalSize=91.9 K 2024-11-19T17:14:26,069 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/74d6483b2b824c53a083b27e44dedecb, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/64e2d1bf96f047f38da96274db5624ae, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/946da17212fa4adb8844b9f6e72179ea] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp, totalSize=36.4 K 2024-11-19T17:14:26,069 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:26,069 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. files: [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/4d7bd0e29f4c4004bb2a120f0471eafb, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/e17196e1c4d941e89b4cc6fc998a328b, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/ed214672a3774e048dce0f4e6c8c2315] 2024-11-19T17:14:26,069 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 74d6483b2b824c53a083b27e44dedecb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732036464637 2024-11-19T17:14:26,069 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d7bd0e29f4c4004bb2a120f0471eafb, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732036464637 2024-11-19T17:14:26,070 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 64e2d1bf96f047f38da96274db5624ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732036464708 2024-11-19T17:14:26,070 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting e17196e1c4d941e89b4cc6fc998a328b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732036464708 2024-11-19T17:14:26,070 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 946da17212fa4adb8844b9f6e72179ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732036464833 2024-11-19T17:14:26,071 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed214672a3774e048dce0f4e6c8c2315, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732036464833 2024-11-19T17:14:26,089 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d916b1a7715625884544afede338b7a#B#compaction#199 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:26,090 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/038f89d119c143ee9921a205831ec2ca is 50, key is test_row_0/B:col10/1732036464833/Put/seqid=0 2024-11-19T17:14:26,099 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:26,119 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119a873e100339f40c2ab4f7f9970b9400c_5d916b1a7715625884544afede338b7a store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:26,122 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119a873e100339f40c2ab4f7f9970b9400c_5d916b1a7715625884544afede338b7a, store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:26,122 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a873e100339f40c2ab4f7f9970b9400c_5d916b1a7715625884544afede338b7a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:26,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T17:14:26,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:26,138 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 5d916b1a7715625884544afede338b7a 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-19T17:14:26,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=A 2024-11-19T17:14:26,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:26,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=B 2024-11-19T17:14:26,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:26,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=C 2024-11-19T17:14:26,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:26,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742056_1232 (size=12881) 2024-11-19T17:14:26,161 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/038f89d119c143ee9921a205831ec2ca as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/038f89d119c143ee9921a205831ec2ca 2024-11-19T17:14:26,168 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d916b1a7715625884544afede338b7a/B of 5d916b1a7715625884544afede338b7a into 038f89d119c143ee9921a205831ec2ca(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:26,168 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:26,168 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., storeName=5d916b1a7715625884544afede338b7a/B, priority=13, startTime=1732036466067; duration=0sec 2024-11-19T17:14:26,168 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:26,168 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d916b1a7715625884544afede338b7a:B 2024-11-19T17:14:26,168 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:26,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ecf3590fd7b34489bee5a001a4b51103_5d916b1a7715625884544afede338b7a is 50, key is test_row_0/A:col10/1732036465556/Put/seqid=0 2024-11-19T17:14:26,176 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:26,176 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 5d916b1a7715625884544afede338b7a/C is initiating minor compaction (all files) 2024-11-19T17:14:26,176 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d916b1a7715625884544afede338b7a/C in TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:26,177 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/21929e2424c74fe684ab756f47962a10, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/4168946b95ca46319c589f7e6fafe7b5, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/08dfd8a435764d06b911e84d05619056] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp, totalSize=36.4 K 2024-11-19T17:14:26,177 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 21929e2424c74fe684ab756f47962a10, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732036464637 2024-11-19T17:14:26,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742058_1234 (size=12454) 2024-11-19T17:14:26,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742057_1233 (size=4469) 2024-11-19T17:14:26,180 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 4168946b95ca46319c589f7e6fafe7b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732036464708 2024-11-19T17:14:26,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:26,180 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 08dfd8a435764d06b911e84d05619056, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732036464833 2024-11-19T17:14:26,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T17:14:26,184 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ecf3590fd7b34489bee5a001a4b51103_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ecf3590fd7b34489bee5a001a4b51103_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:26,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/be74860499714bb2947b347fbc3017e1, store: [table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:26,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/be74860499714bb2947b347fbc3017e1 is 175, key is test_row_0/A:col10/1732036465556/Put/seqid=0 2024-11-19T17:14:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:26,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:26,199 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d916b1a7715625884544afede338b7a#C#compaction#202 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:26,199 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/955e797a398043fdac9cd10e73b195d2 is 50, key is test_row_0/C:col10/1732036464833/Put/seqid=0 2024-11-19T17:14:26,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742059_1235 (size=31255) 2024-11-19T17:14:26,203 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=326, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/be74860499714bb2947b347fbc3017e1 2024-11-19T17:14:26,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/c3785d54940f46059519be9d14228608 is 50, key is test_row_0/B:col10/1732036465556/Put/seqid=0 2024-11-19T17:14:26,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742060_1236 (size=12881) 2024-11-19T17:14:26,225 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/955e797a398043fdac9cd10e73b195d2 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/955e797a398043fdac9cd10e73b195d2 2024-11-19T17:14:26,240 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d916b1a7715625884544afede338b7a/C of 5d916b1a7715625884544afede338b7a into 955e797a398043fdac9cd10e73b195d2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:26,240 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:26,240 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., storeName=5d916b1a7715625884544afede338b7a/C, priority=13, startTime=1732036466067; duration=0sec 2024-11-19T17:14:26,240 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:26,240 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d916b1a7715625884544afede338b7a:C 2024-11-19T17:14:26,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036526207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:26,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:26,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036526241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:26,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:26,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036526241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:26,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:26,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036526241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:26,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:26,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:26,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036526247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:26,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:26,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742061_1237 (size=12301) 2024-11-19T17:14:26,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:26,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036526353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:26,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:26,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036526364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:26,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:26,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:26,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036526364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:26,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:26,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036526364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:26,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:26,549 INFO [master/280a42d09ec8:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-19T17:14:26,549 INFO [master/280a42d09ec8:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-19T17:14:26,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036526574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:26,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036526575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:26,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:26,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:26,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036526575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:26,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:26,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036526575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:26,583 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d916b1a7715625884544afede338b7a#A#compaction#200 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:26,584 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/80ce05b478cf452d8586a27015d8f2bf is 175, key is test_row_0/A:col10/1732036464833/Put/seqid=0 2024-11-19T17:14:26,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742062_1238 (size=31835) 2024-11-19T17:14:26,600 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/80ce05b478cf452d8586a27015d8f2bf as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/80ce05b478cf452d8586a27015d8f2bf 2024-11-19T17:14:26,605 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d916b1a7715625884544afede338b7a/A of 5d916b1a7715625884544afede338b7a into 80ce05b478cf452d8586a27015d8f2bf(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:26,605 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:26,605 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., storeName=5d916b1a7715625884544afede338b7a/A, priority=13, startTime=1732036466067; duration=0sec 2024-11-19T17:14:26,605 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:26,605 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d916b1a7715625884544afede338b7a:A 2024-11-19T17:14:26,654 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/c3785d54940f46059519be9d14228608 2024-11-19T17:14:26,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/55aa9e1f05b34f4e81397bb5f2f8f22a is 50, key is test_row_0/C:col10/1732036465556/Put/seqid=0 2024-11-19T17:14:26,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742063_1239 (size=12301) 2024-11-19T17:14:26,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036526754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:26,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:26,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:26,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036526884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:26,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:26,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036526885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:26,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:26,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036526885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:26,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:26,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:26,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036526885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:26,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:26,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:27,087 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/55aa9e1f05b34f4e81397bb5f2f8f22a 2024-11-19T17:14:27,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/be74860499714bb2947b347fbc3017e1 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/be74860499714bb2947b347fbc3017e1 2024-11-19T17:14:27,104 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/be74860499714bb2947b347fbc3017e1, entries=150, sequenceid=326, filesize=30.5 K 2024-11-19T17:14:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/c3785d54940f46059519be9d14228608 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/c3785d54940f46059519be9d14228608 2024-11-19T17:14:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,111 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/c3785d54940f46059519be9d14228608, entries=150, sequenceid=326, filesize=12.0 K 2024-11-19T17:14:27,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/55aa9e1f05b34f4e81397bb5f2f8f22a as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/55aa9e1f05b34f4e81397bb5f2f8f22a 2024-11-19T17:14:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,117 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/55aa9e1f05b34f4e81397bb5f2f8f22a, entries=150, sequenceid=326, filesize=12.0 K 2024-11-19T17:14:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,118 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 5d916b1a7715625884544afede338b7a in 980ms, sequenceid=326, compaction requested=false 2024-11-19T17:14:27,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:27,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:27,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-19T17:14:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-19T17:14:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-19T17:14:27,123 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0700 sec 2024-11-19T17:14:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,125 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 2.0790 sec 2024-11-19T17:14:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T17:14:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,194 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:14:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,195 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:14:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-19T17:14:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T17:14:27,197 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:14:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,198 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:14:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:14:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T17:14:27,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-19T17:14:27,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:27,351 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 5d916b1a7715625884544afede338b7a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T17:14:27,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=A 2024-11-19T17:14:27,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:27,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=B 2024-11-19T17:14:27,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:27,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=C 2024-11-19T17:14:27,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:27,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194026395cd7534eeeb73c4a5e11927e99_5d916b1a7715625884544afede338b7a is 50, key is test_row_0/A:col10/1732036466207/Put/seqid=0 2024-11-19T17:14:27,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742064_1240 (size=9914) 2024-11-19T17:14:27,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,381 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194026395cd7534eeeb73c4a5e11927e99_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194026395cd7534eeeb73c4a5e11927e99_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/d7d94027d1604df58b35a5c9f59d08e1, store: [table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/d7d94027d1604df58b35a5c9f59d08e1 is 175, key is test_row_0/A:col10/1732036466207/Put/seqid=0 2024-11-19T17:14:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742065_1241 (size=22561) 2024-11-19T17:14:27,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:27,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:27,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:27,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732036467417, max timestamp=1732036467436Num uniques 859; 2024-11-19T17:14:27,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732036467417, max timestamp=1732036467436Num uniques 859; 2024-11-19T17:14:27,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732036467417, max timestamp=1732036467436Num uniques 859; 2024-11-19T17:14:27,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-19T17:14:27,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-19T17:14:27,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-19T17:14:27,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036527457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:27,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:27,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036527459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:27,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:27,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:27,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:27,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:27,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:27,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:27,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036527475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:27,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:27,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:27,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036527476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:27,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:27,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:27,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:27,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:27,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:27,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T17:14:27,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:27,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:27,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036527584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:27,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036527584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:27,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:27,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:27,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:27,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:27,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:27,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:27,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036527593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:27,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036527593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:27,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036527774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:27,776 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_0' in 5d916b1a7715625884544afede338b7a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:14:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:27,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:27,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:27,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036527793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:27,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036527793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:27,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:27,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:27,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:27,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:27,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:27,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:27,796 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=339, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/d7d94027d1604df58b35a5c9f59d08e1 2024-11-19T17:14:27,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:27,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036527804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:27,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/429c3ebc9eb647d7b819b4e52d64efdc is 50, key is test_row_0/B:col10/1732036466207/Put/seqid=0 2024-11-19T17:14:27,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036527805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:27,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742066_1242 (size=9857) 2024-11-19T17:14:27,811 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/429c3ebc9eb647d7b819b4e52d64efdc 2024-11-19T17:14:27,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/1f66f7766abd495a9578c060ca229785 is 50, key is test_row_0/C:col10/1732036466207/Put/seqid=0 2024-11-19T17:14:27,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T17:14:27,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742067_1243 (size=9857) 2024-11-19T17:14:28,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036528104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:28,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:28,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036528104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:28,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:28,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:28,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:28,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036528113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:28,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:28,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036528114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:28,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:28,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:28,229 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/1f66f7766abd495a9578c060ca229785 2024-11-19T17:14:28,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/d7d94027d1604df58b35a5c9f59d08e1 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/d7d94027d1604df58b35a5c9f59d08e1 2024-11-19T17:14:28,246 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/d7d94027d1604df58b35a5c9f59d08e1, entries=100, sequenceid=339, filesize=22.0 K 2024-11-19T17:14:28,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/429c3ebc9eb647d7b819b4e52d64efdc as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/429c3ebc9eb647d7b819b4e52d64efdc 2024-11-19T17:14:28,254 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/429c3ebc9eb647d7b819b4e52d64efdc, entries=100, sequenceid=339, filesize=9.6 K 2024-11-19T17:14:28,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/1f66f7766abd495a9578c060ca229785 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/1f66f7766abd495a9578c060ca229785 2024-11-19T17:14:28,268 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/1f66f7766abd495a9578c060ca229785, entries=100, sequenceid=339, filesize=9.6 K 2024-11-19T17:14:28,269 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for 5d916b1a7715625884544afede338b7a in 918ms, sequenceid=339, compaction requested=true 2024-11-19T17:14:28,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:28,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:28,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-19T17:14:28,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-19T17:14:28,276 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-19T17:14:28,276 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0730 sec 2024-11-19T17:14:28,279 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.0830 sec 2024-11-19T17:14:28,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T17:14:28,334 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:14:28,334 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:14:28,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-19T17:14:28,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T17:14:28,337 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:14:28,337 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:14:28,338 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:14:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T17:14:28,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-19T17:14:28,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:28,492 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 5d916b1a7715625884544afede338b7a 3/3 column families, dataSize=147.60 KB heapSize=401.39 KB 2024-11-19T17:14:28,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=A 2024-11-19T17:14:28,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:28,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=B 2024-11-19T17:14:28,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:28,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=C 2024-11-19T17:14:28,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:28,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191da4d07a19224fff98096bd8161bac5c_5d916b1a7715625884544afede338b7a is 50, key is test_row_0/A:col10/1732036467459/Put/seqid=0 2024-11-19T17:14:28,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742068_1244 (size=12454) 2024-11-19T17:14:28,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:28,521 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191da4d07a19224fff98096bd8161bac5c_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191da4d07a19224fff98096bd8161bac5c_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:28,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/8ea4eb0d84e945b991123934410babbc, store: [table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:28,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/8ea4eb0d84e945b991123934410babbc is 175, key is test_row_0/A:col10/1732036467459/Put/seqid=0 2024-11-19T17:14:28,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742069_1245 (size=31255) 2024-11-19T17:14:28,536 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=373, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/8ea4eb0d84e945b991123934410babbc 2024-11-19T17:14:28,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/5933c0f2cdd04da3860452449b3e1d66 is 50, key is test_row_0/B:col10/1732036467459/Put/seqid=0 2024-11-19T17:14:28,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742070_1246 (size=12301) 2024-11-19T17:14:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:28,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:28,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036528632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:28,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:28,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036528634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:28,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:28,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036528638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:28,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:28,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036528639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T17:14:28,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036528744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:28,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:28,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036528744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:28,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:28,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:28,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036528744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:28,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:28,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036528753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:28,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:28,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:28,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036528954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:28,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:28,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:28,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036528954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:28,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:28,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:28,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036528954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:28,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:28,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T17:14:28,969 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/5933c0f2cdd04da3860452449b3e1d66 2024-11-19T17:14:28,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:28,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036528973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:28,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:28,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:28,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/7d7c67bbc01c4400a3587c9899558bf7 is 50, key is test_row_0/C:col10/1732036467459/Put/seqid=0 2024-11-19T17:14:28,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742071_1247 (size=12301) 2024-11-19T17:14:28,988 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/7d7c67bbc01c4400a3587c9899558bf7 2024-11-19T17:14:28,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/8ea4eb0d84e945b991123934410babbc as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/8ea4eb0d84e945b991123934410babbc 2024-11-19T17:14:29,006 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/8ea4eb0d84e945b991123934410babbc, entries=150, sequenceid=373, filesize=30.5 K 2024-11-19T17:14:29,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/5933c0f2cdd04da3860452449b3e1d66 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/5933c0f2cdd04da3860452449b3e1d66 2024-11-19T17:14:29,012 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/5933c0f2cdd04da3860452449b3e1d66, entries=150, sequenceid=373, filesize=12.0 K 2024-11-19T17:14:29,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/7d7c67bbc01c4400a3587c9899558bf7 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/7d7c67bbc01c4400a3587c9899558bf7 2024-11-19T17:14:29,018 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/7d7c67bbc01c4400a3587c9899558bf7, entries=150, sequenceid=373, filesize=12.0 K 2024-11-19T17:14:29,019 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~401.34 KB/410976, currentSize=46.96 KB/48090 for 5d916b1a7715625884544afede338b7a in 527ms, sequenceid=373, compaction requested=true 2024-11-19T17:14:29,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:29,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:29,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-19T17:14:29,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-19T17:14:29,023 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-19T17:14:29,023 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 684 msec 2024-11-19T17:14:29,025 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 689 msec 2024-11-19T17:14:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:29,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d916b1a7715625884544afede338b7a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T17:14:29,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=A 2024-11-19T17:14:29,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:29,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=B 2024-11-19T17:14:29,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:29,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=C 2024-11-19T17:14:29,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:29,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192ae2c42a499b48a0b49966d407f0e55f_5d916b1a7715625884544afede338b7a is 50, key is test_row_0/A:col10/1732036469265/Put/seqid=0 2024-11-19T17:14:29,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742072_1248 (size=14994) 2024-11-19T17:14:29,304 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:29,310 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192ae2c42a499b48a0b49966d407f0e55f_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192ae2c42a499b48a0b49966d407f0e55f_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:29,312 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/4f549abb53ee49a0895a6cb520f87f1b, store: [table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:29,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/4f549abb53ee49a0895a6cb520f87f1b is 175, key is test_row_0/A:col10/1732036469265/Put/seqid=0 2024-11-19T17:14:29,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732036469290, max timestamp=1732036469311Num uniques 859; 2024-11-19T17:14:29,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-19T17:14:29,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036469293, max timestamp=1732036469314Num uniques 859; 2024-11-19T17:14:29,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:14:29,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036469293, max timestamp=1732036469314Num uniques 859; 2024-11-19T17:14:29,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:14:29,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742073_1249 (size=39949) 2024-11-19T17:14:29,331 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=384, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/4f549abb53ee49a0895a6cb520f87f1b 2024-11-19T17:14:29,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/7940786ceda74b02b08ea0941e73f2b3 is 50, key is test_row_0/B:col10/1732036469265/Put/seqid=0 2024-11-19T17:14:29,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742074_1250 (size=12301) 2024-11-19T17:14:29,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/7940786ceda74b02b08ea0941e73f2b3 2024-11-19T17:14:29,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036529348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:29,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036529349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:29,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:29,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:29,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/2c71e965a752452384fafb7f3b4d8c78 is 50, key is test_row_0/C:col10/1732036469265/Put/seqid=0 2024-11-19T17:14:29,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036529365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:29,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:29,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036529367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:29,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:29,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742075_1251 (size=12301) 2024-11-19T17:14:29,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/2c71e965a752452384fafb7f3b4d8c78 2024-11-19T17:14:29,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/4f549abb53ee49a0895a6cb520f87f1b as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/4f549abb53ee49a0895a6cb520f87f1b 2024-11-19T17:14:29,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/4f549abb53ee49a0895a6cb520f87f1b, entries=200, sequenceid=384, filesize=39.0 K 2024-11-19T17:14:29,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/7940786ceda74b02b08ea0941e73f2b3 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/7940786ceda74b02b08ea0941e73f2b3 2024-11-19T17:14:29,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/7940786ceda74b02b08ea0941e73f2b3, entries=150, sequenceid=384, filesize=12.0 K 2024-11-19T17:14:29,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/2c71e965a752452384fafb7f3b4d8c78 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/2c71e965a752452384fafb7f3b4d8c78 2024-11-19T17:14:29,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/2c71e965a752452384fafb7f3b4d8c78, entries=150, sequenceid=384, filesize=12.0 K 2024-11-19T17:14:29,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=138.65 KB/141980 for 5d916b1a7715625884544afede338b7a in 151ms, sequenceid=384, compaction requested=true 2024-11-19T17:14:29,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:29,418 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T17:14:29,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d916b1a7715625884544afede338b7a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:14:29,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:29,419 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T17:14:29,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d916b1a7715625884544afede338b7a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:14:29,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:29,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d916b1a7715625884544afede338b7a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:14:29,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:29,432 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 156855 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T17:14:29,432 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 5d916b1a7715625884544afede338b7a/A is initiating minor compaction (all files) 2024-11-19T17:14:29,432 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d916b1a7715625884544afede338b7a/A in TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:29,432 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/80ce05b478cf452d8586a27015d8f2bf, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/be74860499714bb2947b347fbc3017e1, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/d7d94027d1604df58b35a5c9f59d08e1, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/8ea4eb0d84e945b991123934410babbc, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/4f549abb53ee49a0895a6cb520f87f1b] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp, totalSize=153.2 K 2024-11-19T17:14:29,433 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:29,433 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. files: [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/80ce05b478cf452d8586a27015d8f2bf, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/be74860499714bb2947b347fbc3017e1, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/d7d94027d1604df58b35a5c9f59d08e1, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/8ea4eb0d84e945b991123934410babbc, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/4f549abb53ee49a0895a6cb520f87f1b] 2024-11-19T17:14:29,433 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80ce05b478cf452d8586a27015d8f2bf, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732036464833 2024-11-19T17:14:29,433 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 59641 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T17:14:29,434 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 5d916b1a7715625884544afede338b7a/B is initiating minor compaction (all files) 2024-11-19T17:14:29,434 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d916b1a7715625884544afede338b7a/B in TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:29,434 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/038f89d119c143ee9921a205831ec2ca, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/c3785d54940f46059519be9d14228608, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/429c3ebc9eb647d7b819b4e52d64efdc, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/5933c0f2cdd04da3860452449b3e1d66, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/7940786ceda74b02b08ea0941e73f2b3] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp, totalSize=58.2 K 2024-11-19T17:14:29,434 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting be74860499714bb2947b347fbc3017e1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732036465556 2024-11-19T17:14:29,436 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7d94027d1604df58b35a5c9f59d08e1, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732036466207 2024-11-19T17:14:29,437 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ea4eb0d84e945b991123934410babbc, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732036467456 2024-11-19T17:14:29,437 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 038f89d119c143ee9921a205831ec2ca, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732036464833 2024-11-19T17:14:29,438 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f549abb53ee49a0895a6cb520f87f1b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732036468632 2024-11-19T17:14:29,438 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting c3785d54940f46059519be9d14228608, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732036465556 2024-11-19T17:14:29,441 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 429c3ebc9eb647d7b819b4e52d64efdc, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732036466207 2024-11-19T17:14:29,442 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 5933c0f2cdd04da3860452449b3e1d66, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732036467456 2024-11-19T17:14:29,442 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 7940786ceda74b02b08ea0941e73f2b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732036468632 2024-11-19T17:14:29,454 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:29,465 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119cb5075b115ea4e86a5c25ec0882be2e1_5d916b1a7715625884544afede338b7a store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:29,468 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119cb5075b115ea4e86a5c25ec0882be2e1_5d916b1a7715625884544afede338b7a, store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:29,469 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119cb5075b115ea4e86a5c25ec0882be2e1_5d916b1a7715625884544afede338b7a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:29,471 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d916b1a7715625884544afede338b7a#B#compaction#215 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:29,472 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/e8b8e689a7224deaa457a1d96bece33e is 50, key is test_row_0/B:col10/1732036469265/Put/seqid=0 2024-11-19T17:14:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T17:14:29,475 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:14:29,476 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:14:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-19T17:14:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T17:14:29,480 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:14:29,482 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:14:29,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d916b1a7715625884544afede338b7a 3/3 column families, dataSize=145.36 KB heapSize=410.61 KB 2024-11-19T17:14:29,482 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:14:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:29,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=A 2024-11-19T17:14:29,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T17:14:29,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=B 2024-11-19T17:14:29,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T17:14:29,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=C 2024-11-19T17:14:29,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T17:14:29,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742076_1252 (size=4469) 2024-11-19T17:14:29,499 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d916b1a7715625884544afede338b7a#A#compaction#214 average throughput is 0.54 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:29,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036529487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:29,500 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/d24b96ac00de49e8ac47feff9ce0fb3d is 175, key is test_row_0/A:col10/1732036469265/Put/seqid=0 2024-11-19T17:14:29,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:29,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036529496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:29,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742077_1253 (size=13051) 2024-11-19T17:14:29,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036529498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:29,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:29,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:29,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036529499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:29,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:29,516 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/e8b8e689a7224deaa457a1d96bece33e as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/e8b8e689a7224deaa457a1d96bece33e 2024-11-19T17:14:29,523 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5d916b1a7715625884544afede338b7a/B of 5d916b1a7715625884544afede338b7a into e8b8e689a7224deaa457a1d96bece33e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:29,523 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:29,523 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., storeName=5d916b1a7715625884544afede338b7a/B, priority=11, startTime=1732036469419; duration=0sec 2024-11-19T17:14:29,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119efe03ddddc744d7d867d7f1daeedd3ab_5d916b1a7715625884544afede338b7a is 50, key is test_row_0/A:col10/1732036469481/Put/seqid=0 2024-11-19T17:14:29,523 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:29,523 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d916b1a7715625884544afede338b7a:B 2024-11-19T17:14:29,524 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T17:14:29,527 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 59641 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T17:14:29,527 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 5d916b1a7715625884544afede338b7a/C is initiating minor compaction (all files) 2024-11-19T17:14:29,527 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d916b1a7715625884544afede338b7a/C in TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:29,527 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/955e797a398043fdac9cd10e73b195d2, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/55aa9e1f05b34f4e81397bb5f2f8f22a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/1f66f7766abd495a9578c060ca229785, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/7d7c67bbc01c4400a3587c9899558bf7, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/2c71e965a752452384fafb7f3b4d8c78] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp, totalSize=58.2 K 2024-11-19T17:14:29,528 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 955e797a398043fdac9cd10e73b195d2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732036464833 2024-11-19T17:14:29,529 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 55aa9e1f05b34f4e81397bb5f2f8f22a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732036465556 2024-11-19T17:14:29,529 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f66f7766abd495a9578c060ca229785, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732036466207 2024-11-19T17:14:29,530 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d7c67bbc01c4400a3587c9899558bf7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732036467456 2024-11-19T17:14:29,530 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c71e965a752452384fafb7f3b4d8c78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732036468632 2024-11-19T17:14:29,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742078_1254 (size=32005) 2024-11-19T17:14:29,561 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/d24b96ac00de49e8ac47feff9ce0fb3d as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/d24b96ac00de49e8ac47feff9ce0fb3d 2024-11-19T17:14:29,566 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d916b1a7715625884544afede338b7a#C#compaction#217 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:29,567 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/e84243131e9a48268f6ce7edd8ed8ff3 is 50, key is test_row_0/C:col10/1732036469265/Put/seqid=0 2024-11-19T17:14:29,572 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5d916b1a7715625884544afede338b7a/A of 5d916b1a7715625884544afede338b7a into d24b96ac00de49e8ac47feff9ce0fb3d(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:29,572 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:29,572 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., storeName=5d916b1a7715625884544afede338b7a/A, priority=11, startTime=1732036469418; duration=0sec 2024-11-19T17:14:29,572 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:29,572 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d916b1a7715625884544afede338b7a:A 2024-11-19T17:14:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T17:14:29,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742079_1255 (size=12454) 2024-11-19T17:14:29,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036529603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:29,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742080_1256 (size=13051) 2024-11-19T17:14:29,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:29,619 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/e84243131e9a48268f6ce7edd8ed8ff3 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/e84243131e9a48268f6ce7edd8ed8ff3 2024-11-19T17:14:29,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036529614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:29,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036529614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:29,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036529614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:29,625 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5d916b1a7715625884544afede338b7a/C of 5d916b1a7715625884544afede338b7a into e84243131e9a48268f6ce7edd8ed8ff3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:29,625 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:29,625 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., storeName=5d916b1a7715625884544afede338b7a/C, priority=11, startTime=1732036469419; duration=0sec 2024-11-19T17:14:29,625 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:29,625 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d916b1a7715625884544afede338b7a:C 2024-11-19T17:14:29,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T17:14:29,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:29,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:29,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:29,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:29,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:29,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036529785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:29,786 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_0' in 5d916b1a7715625884544afede338b7a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:14:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:29,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T17:14:29,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:29,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:29,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:29,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:29,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:29,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:29,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T17:14:29,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:29,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036529814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:29,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:29,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:29,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036529834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:29,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:29,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036529834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:29,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:29,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:29,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:29,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036529834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:29,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:29,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:29,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T17:14:29,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:29,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:29,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:29,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:29,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:29,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:29,993 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:30,005 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119efe03ddddc744d7d867d7f1daeedd3ab_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119efe03ddddc744d7d867d7f1daeedd3ab_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:30,006 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/38ba54e3a0f14202bbb7dafb289beed4, store: [table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:30,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/38ba54e3a0f14202bbb7dafb289beed4 is 175, key is test_row_0/A:col10/1732036469481/Put/seqid=0 2024-11-19T17:14:30,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742081_1257 (size=31255) 2024-11-19T17:14:30,025 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=422, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/38ba54e3a0f14202bbb7dafb289beed4 2024-11-19T17:14:30,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/67e5ace200564580a1ddf8fdc5884f10 is 50, key is test_row_0/B:col10/1732036469481/Put/seqid=0 2024-11-19T17:14:30,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742082_1258 (size=12301) 2024-11-19T17:14:30,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/67e5ace200564580a1ddf8fdc5884f10 2024-11-19T17:14:30,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/6193bd9952404e0b81635fce6460f1d5 is 50, key is test_row_0/C:col10/1732036469481/Put/seqid=0 2024-11-19T17:14:30,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742083_1259 (size=12301) 2024-11-19T17:14:30,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T17:14:30,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:30,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:30,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:30,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:30,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:30,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:30,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T17:14:30,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:30,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036530124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:30,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:30,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:30,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:30,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:30,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036530143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:30,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:30,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036530143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:30,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:30,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:30,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:30,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036530144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:30,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:30,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:30,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:30,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:30,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T17:14:30,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:30,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:30,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:30,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:30,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:30,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:30,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T17:14:30,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:30,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:30,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:30,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:30,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:30,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:30,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/6193bd9952404e0b81635fce6460f1d5 2024-11-19T17:14:30,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/38ba54e3a0f14202bbb7dafb289beed4 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/38ba54e3a0f14202bbb7dafb289beed4 2024-11-19T17:14:30,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/38ba54e3a0f14202bbb7dafb289beed4, entries=150, sequenceid=422, filesize=30.5 K 2024-11-19T17:14:30,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/67e5ace200564580a1ddf8fdc5884f10 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/67e5ace200564580a1ddf8fdc5884f10 2024-11-19T17:14:30,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/67e5ace200564580a1ddf8fdc5884f10, entries=150, sequenceid=422, filesize=12.0 K 2024-11-19T17:14:30,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/6193bd9952404e0b81635fce6460f1d5 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/6193bd9952404e0b81635fce6460f1d5 2024-11-19T17:14:30,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/6193bd9952404e0b81635fce6460f1d5, entries=150, sequenceid=422, filesize=12.0 K 2024-11-19T17:14:30,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~152.07 KB/155720, heapSize ~430.95 KB/441296, currentSize=33.54 KB/34350 for 5d916b1a7715625884544afede338b7a in 1046ms, sequenceid=422, compaction requested=false 2024-11-19T17:14:30,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:30,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T17:14:30,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:30,560 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 5d916b1a7715625884544afede338b7a 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T17:14:30,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=A 2024-11-19T17:14:30,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:30,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=B 2024-11-19T17:14:30,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:30,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=C 2024-11-19T17:14:30,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:30,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f56151e64e894e898b2e54931597c673_5d916b1a7715625884544afede338b7a is 50, key is test_row_0/A:col10/1732036469497/Put/seqid=0 2024-11-19T17:14:30,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742084_1260 (size=12454) 2024-11-19T17:14:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T17:14:30,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:30,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:30,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732036470660, max timestamp=1732036470683Num uniques 859; 2024-11-19T17:14:30,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T17:14:30,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:30,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036530699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:30,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:30,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036530700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:30,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:30,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036530705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:30,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:30,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:30,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:30,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:30,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:30,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:30,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:30,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:30,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:30,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:30,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036530707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:30,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:30,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036530813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:30,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:30,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036530814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:30,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:30,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:30,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:30,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:30,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036530814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:30,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:30,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036530824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:30,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:30,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:30,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:31,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:31,012 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f56151e64e894e898b2e54931597c673_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f56151e64e894e898b2e54931597c673_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:31,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/7639f21e5d574b278daa1f1ae6bc5440, store: [table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:31,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/7639f21e5d574b278daa1f1ae6bc5440 is 175, key is test_row_0/A:col10/1732036469497/Put/seqid=0 2024-11-19T17:14:31,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742085_1261 (size=31255) 2024-11-19T17:14:31,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036531024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:31,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:31,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036531024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:31,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:31,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:31,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036531024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:31,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:31,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036531033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:31,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:31,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036531334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036531334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:31,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:31,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036531334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:31,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:31,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036531343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:31,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:31,418 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=433, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/7639f21e5d574b278daa1f1ae6bc5440 2024-11-19T17:14:31,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/0e3a7e9baa9441f18cd60a9efce910dd is 50, key is test_row_0/B:col10/1732036469497/Put/seqid=0 2024-11-19T17:14:31,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742086_1262 (size=12301) 2024-11-19T17:14:31,435 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/0e3a7e9baa9441f18cd60a9efce910dd 2024-11-19T17:14:31,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/1e1551c8e0384da697ef1fd047bb4de4 is 50, key is test_row_0/C:col10/1732036469497/Put/seqid=0 2024-11-19T17:14:31,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742087_1263 (size=12301) 2024-11-19T17:14:31,454 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/1e1551c8e0384da697ef1fd047bb4de4 2024-11-19T17:14:31,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/7639f21e5d574b278daa1f1ae6bc5440 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/7639f21e5d574b278daa1f1ae6bc5440 2024-11-19T17:14:31,468 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/7639f21e5d574b278daa1f1ae6bc5440, entries=150, sequenceid=433, filesize=30.5 K 2024-11-19T17:14:31,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/0e3a7e9baa9441f18cd60a9efce910dd as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/0e3a7e9baa9441f18cd60a9efce910dd 2024-11-19T17:14:31,480 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/0e3a7e9baa9441f18cd60a9efce910dd, entries=150, sequenceid=433, filesize=12.0 K 2024-11-19T17:14:31,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/1e1551c8e0384da697ef1fd047bb4de4 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/1e1551c8e0384da697ef1fd047bb4de4 2024-11-19T17:14:31,490 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/1e1551c8e0384da697ef1fd047bb4de4, entries=150, sequenceid=433, filesize=12.0 K 2024-11-19T17:14:31,491 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=145.36 KB/148850 for 5d916b1a7715625884544afede338b7a in 931ms, sequenceid=433, compaction requested=true 2024-11-19T17:14:31,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:31,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:31,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-19T17:14:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-19T17:14:31,496 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-19T17:14:31,496 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0110 sec 2024-11-19T17:14:31,499 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.0210 sec 2024-11-19T17:14:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T17:14:31,624 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:14:31,625 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:14:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-19T17:14:31,627 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:14:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T17:14:31,628 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:14:31,628 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:14:31,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T17:14:31,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-19T17:14:31,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:31,781 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 5d916b1a7715625884544afede338b7a 3/3 column families, dataSize=145.36 KB heapSize=421.88 KB 2024-11-19T17:14:31,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=A 2024-11-19T17:14:31,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:31,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=B 2024-11-19T17:14:31,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:31,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=C 2024-11-19T17:14:31,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:31,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111992ef67896e644a72b993c5845d8bac71_5d916b1a7715625884544afede338b7a is 50, key is test_row_0/A:col10/1732036470704/Put/seqid=0 2024-11-19T17:14:31,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742088_1264 (size=12454) 2024-11-19T17:14:31,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:31,808 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111992ef67896e644a72b993c5845d8bac71_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111992ef67896e644a72b993c5845d8bac71_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:31,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/05aabb1b3bca449388e24d29301e04a9, store: [table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:31,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/05aabb1b3bca449388e24d29301e04a9 is 175, key is test_row_0/A:col10/1732036470704/Put/seqid=0 2024-11-19T17:14:31,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742089_1265 (size=31255) 2024-11-19T17:14:31,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:31,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:31,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:31,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:31,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036531859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:31,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036531859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:31,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:31,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036531860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:31,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:31,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:31,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:31,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:31,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036531866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:31,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:31,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T17:14:31,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:31,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036531974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:31,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:31,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036531974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:31,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:31,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036531974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:31,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:31,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:31,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:31,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:31,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036531986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:31,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:31,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:32,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:32,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036532194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:32,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:32,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:32,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036532195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:32,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:32,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036532195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:32,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:32,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036532195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:32,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:32,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:32,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:32,225 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=461, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/05aabb1b3bca449388e24d29301e04a9 2024-11-19T17:14:32,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/e61dcf9e2c9841f383db46e057322863 is 50, key is test_row_0/B:col10/1732036470704/Put/seqid=0 2024-11-19T17:14:32,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T17:14:32,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742090_1266 (size=12301) 2024-11-19T17:14:32,262 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/e61dcf9e2c9841f383db46e057322863 2024-11-19T17:14:32,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/c9ecb62cb8f2440a8cbce2ecca81b1d5 is 50, key is test_row_0/C:col10/1732036470704/Put/seqid=0 2024-11-19T17:14:32,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742091_1267 (size=12301) 2024-11-19T17:14:32,298 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/c9ecb62cb8f2440a8cbce2ecca81b1d5 2024-11-19T17:14:32,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/05aabb1b3bca449388e24d29301e04a9 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/05aabb1b3bca449388e24d29301e04a9 2024-11-19T17:14:32,312 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/05aabb1b3bca449388e24d29301e04a9, entries=150, sequenceid=461, filesize=30.5 K 2024-11-19T17:14:32,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/e61dcf9e2c9841f383db46e057322863 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/e61dcf9e2c9841f383db46e057322863 2024-11-19T17:14:32,320 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/e61dcf9e2c9841f383db46e057322863, entries=150, sequenceid=461, filesize=12.0 K 2024-11-19T17:14:32,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/c9ecb62cb8f2440a8cbce2ecca81b1d5 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/c9ecb62cb8f2440a8cbce2ecca81b1d5 2024-11-19T17:14:32,326 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/c9ecb62cb8f2440a8cbce2ecca81b1d5, entries=150, sequenceid=461, filesize=12.0 K 2024-11-19T17:14:32,327 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~145.36 KB/148850, heapSize ~421.83 KB/431952, currentSize=33.54 KB/34350 for 5d916b1a7715625884544afede338b7a in 546ms, sequenceid=461, compaction requested=true 2024-11-19T17:14:32,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:32,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:32,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-19T17:14:32,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-19T17:14:32,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-19T17:14:32,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 701 msec 2024-11-19T17:14:32,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 710 msec 2024-11-19T17:14:32,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:32,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d916b1a7715625884544afede338b7a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T17:14:32,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=A 2024-11-19T17:14:32,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:32,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=B 2024-11-19T17:14:32,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:32,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=C 2024-11-19T17:14:32,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:32,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198aa3245e4c7d409aaefcef848ef8dbda_5d916b1a7715625884544afede338b7a is 50, key is test_row_0/A:col10/1732036471856/Put/seqid=0 2024-11-19T17:14:32,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742092_1268 (size=14994) 2024-11-19T17:14:32,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036472544, max timestamp=1732036472580Num uniques 859; 2024-11-19T17:14:32,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:14:32,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036472544, max timestamp=1732036472580Num uniques 859; 2024-11-19T17:14:32,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036472544, max timestamp=1732036472580Num uniques 859; 2024-11-19T17:14:32,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:14:32,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:14:32,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:32,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036532627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:32,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:32,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036532628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:32,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:32,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:32,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:32,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036532628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:32,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:32,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036532629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:32,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:32,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:32,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:32,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036532744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:32,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:32,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036532744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:32,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:32,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:32,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:32,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036532744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:32,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:32,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036532744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:32,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:32,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:32,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T17:14:32,764 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:14:32,764 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:14:32,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-19T17:14:32,767 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:14:32,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T17:14:32,768 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:14:32,768 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:14:32,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T17:14:32,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T17:14:32,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:32,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:32,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:32,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:32,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:32,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:32,934 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:32,940 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198aa3245e4c7d409aaefcef848ef8dbda_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198aa3245e4c7d409aaefcef848ef8dbda_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:32,941 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/6cb4d2f0ff994c4e8d45b20e402bf634, store: [table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:32,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/6cb4d2f0ff994c4e8d45b20e402bf634 is 175, key is test_row_0/A:col10/1732036471856/Put/seqid=0 2024-11-19T17:14:32,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:32,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036532954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:32,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:32,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036532954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:32,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:32,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:32,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036532955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:32,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:32,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:32,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036532961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:32,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:32,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:32,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742093_1269 (size=39949) 2024-11-19T17:14:33,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T17:14:33,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:33,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:33,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:33,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:33,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:33,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T17:14:33,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:33,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T17:14:33,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:33,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:33,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:33,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:33,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:33,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:33,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:33,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036533263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:33,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:33,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:33,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:33,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:33,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036533263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:33,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:33,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:33,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:33,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036533264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:33,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:33,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:33,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:33,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:33,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:33,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036533274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:33,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:33,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:33,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:33,385 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=472, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/6cb4d2f0ff994c4e8d45b20e402bf634 2024-11-19T17:14:33,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T17:14:33,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:33,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:33,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:33,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:33,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:33,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T17:14:33,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:33,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/52a19c7046034776a8af5330060a6bec is 50, key is test_row_0/B:col10/1732036471856/Put/seqid=0 2024-11-19T17:14:33,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742094_1270 (size=12301) 2024-11-19T17:14:33,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/52a19c7046034776a8af5330060a6bec 2024-11-19T17:14:33,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/dfe2722f821d4d63860f170c16b160ff is 50, key is test_row_0/C:col10/1732036471856/Put/seqid=0 2024-11-19T17:14:33,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742095_1271 (size=12301) 2024-11-19T17:14:33,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T17:14:33,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:33,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:33,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:33,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:33,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:33,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:33,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T17:14:33,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:33,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:33,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:33,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:33,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:33,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:33,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:33,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036533784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:33,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:33,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036533784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:33,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:33,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036533784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:33,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:33,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:33,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:33,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:33,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34222 deadline: 1732036533813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:33,816 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_0' in 5d916b1a7715625884544afede338b7a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:14:33,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:33,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:33,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:33,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:33,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036533824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:33,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:33,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:33,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:33,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T17:14:33,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:33,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:33,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:33,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:33,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:33,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:33,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/dfe2722f821d4d63860f170c16b160ff 2024-11-19T17:14:33,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T17:14:33,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/6cb4d2f0ff994c4e8d45b20e402bf634 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/6cb4d2f0ff994c4e8d45b20e402bf634 2024-11-19T17:14:33,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/6cb4d2f0ff994c4e8d45b20e402bf634, entries=200, sequenceid=472, filesize=39.0 K 2024-11-19T17:14:33,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/52a19c7046034776a8af5330060a6bec as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/52a19c7046034776a8af5330060a6bec 2024-11-19T17:14:33,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/52a19c7046034776a8af5330060a6bec, entries=150, sequenceid=472, filesize=12.0 K 2024-11-19T17:14:33,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/dfe2722f821d4d63860f170c16b160ff as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/dfe2722f821d4d63860f170c16b160ff 2024-11-19T17:14:33,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/dfe2722f821d4d63860f170c16b160ff, entries=150, sequenceid=472, filesize=12.0 K 2024-11-19T17:14:33,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 5d916b1a7715625884544afede338b7a in 1441ms, sequenceid=472, compaction requested=true 2024-11-19T17:14:33,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:33,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d916b1a7715625884544afede338b7a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:14:33,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:33,951 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T17:14:33,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d916b1a7715625884544afede338b7a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:14:33,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:33,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d916b1a7715625884544afede338b7a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:14:33,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:33,951 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T17:14:33,953 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 165719 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T17:14:33,953 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 5d916b1a7715625884544afede338b7a/A is initiating minor compaction (all files) 2024-11-19T17:14:33,953 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d916b1a7715625884544afede338b7a/A in TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:33,954 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/d24b96ac00de49e8ac47feff9ce0fb3d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/38ba54e3a0f14202bbb7dafb289beed4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/7639f21e5d574b278daa1f1ae6bc5440, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/05aabb1b3bca449388e24d29301e04a9, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/6cb4d2f0ff994c4e8d45b20e402bf634] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp, totalSize=161.8 K 2024-11-19T17:14:33,954 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:33,954 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. files: [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/d24b96ac00de49e8ac47feff9ce0fb3d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/38ba54e3a0f14202bbb7dafb289beed4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/7639f21e5d574b278daa1f1ae6bc5440, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/05aabb1b3bca449388e24d29301e04a9, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/6cb4d2f0ff994c4e8d45b20e402bf634] 2024-11-19T17:14:33,955 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting d24b96ac00de49e8ac47feff9ce0fb3d, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732036468632 2024-11-19T17:14:33,955 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38ba54e3a0f14202bbb7dafb289beed4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1732036469363 2024-11-19T17:14:33,956 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7639f21e5d574b278daa1f1ae6bc5440, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732036469483 2024-11-19T17:14:33,956 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62255 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T17:14:33,956 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 5d916b1a7715625884544afede338b7a/B is initiating minor compaction (all files) 2024-11-19T17:14:33,956 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d916b1a7715625884544afede338b7a/B in TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:33,956 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/e8b8e689a7224deaa457a1d96bece33e, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/67e5ace200564580a1ddf8fdc5884f10, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/0e3a7e9baa9441f18cd60a9efce910dd, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/e61dcf9e2c9841f383db46e057322863, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/52a19c7046034776a8af5330060a6bec] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp, totalSize=60.8 K 2024-11-19T17:14:33,956 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05aabb1b3bca449388e24d29301e04a9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732036470696 2024-11-19T17:14:33,957 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting e8b8e689a7224deaa457a1d96bece33e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732036468632 2024-11-19T17:14:33,957 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cb4d2f0ff994c4e8d45b20e402bf634, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1732036471856 2024-11-19T17:14:33,957 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 67e5ace200564580a1ddf8fdc5884f10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1732036469363 2024-11-19T17:14:33,958 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e3a7e9baa9441f18cd60a9efce910dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732036469483 2024-11-19T17:14:33,959 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting e61dcf9e2c9841f383db46e057322863, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732036470696 2024-11-19T17:14:33,959 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 52a19c7046034776a8af5330060a6bec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1732036471856 2024-11-19T17:14:33,981 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:33,994 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d916b1a7715625884544afede338b7a#B#compaction#230 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:33,995 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/1cf96c58b0074a76bac9fa75ae511aa6 is 50, key is test_row_0/B:col10/1732036471856/Put/seqid=0 2024-11-19T17:14:34,003 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111982ac049040f543ee9cf532436d4fbc18_5d916b1a7715625884544afede338b7a store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:34,006 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111982ac049040f543ee9cf532436d4fbc18_5d916b1a7715625884544afede338b7a, store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:34,006 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111982ac049040f543ee9cf532436d4fbc18_5d916b1a7715625884544afede338b7a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:34,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T17:14:34,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:34,010 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 5d916b1a7715625884544afede338b7a 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-19T17:14:34,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=A 2024-11-19T17:14:34,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:34,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=B 2024-11-19T17:14:34,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:34,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=C 2024-11-19T17:14:34,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:34,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742096_1272 (size=13221) 2024-11-19T17:14:34,073 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/1cf96c58b0074a76bac9fa75ae511aa6 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/1cf96c58b0074a76bac9fa75ae511aa6 2024-11-19T17:14:34,079 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5d916b1a7715625884544afede338b7a/B of 5d916b1a7715625884544afede338b7a into 1cf96c58b0074a76bac9fa75ae511aa6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:34,079 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:34,079 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., storeName=5d916b1a7715625884544afede338b7a/B, priority=11, startTime=1732036473951; duration=0sec 2024-11-19T17:14:34,079 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:34,079 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d916b1a7715625884544afede338b7a:B 2024-11-19T17:14:34,079 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T17:14:34,081 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62255 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T17:14:34,081 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 5d916b1a7715625884544afede338b7a/C is initiating minor compaction (all files) 2024-11-19T17:14:34,081 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d916b1a7715625884544afede338b7a/C in TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:34,081 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/e84243131e9a48268f6ce7edd8ed8ff3, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/6193bd9952404e0b81635fce6460f1d5, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/1e1551c8e0384da697ef1fd047bb4de4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/c9ecb62cb8f2440a8cbce2ecca81b1d5, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/dfe2722f821d4d63860f170c16b160ff] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp, totalSize=60.8 K 2024-11-19T17:14:34,082 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting e84243131e9a48268f6ce7edd8ed8ff3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732036468632 2024-11-19T17:14:34,084 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 6193bd9952404e0b81635fce6460f1d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1732036469363 2024-11-19T17:14:34,085 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e1551c8e0384da697ef1fd047bb4de4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732036469483 2024-11-19T17:14:34,086 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting c9ecb62cb8f2440a8cbce2ecca81b1d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732036470696 2024-11-19T17:14:34,086 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting dfe2722f821d4d63860f170c16b160ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1732036471856 2024-11-19T17:14:34,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742097_1273 (size=4469) 2024-11-19T17:14:34,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119824b170d7d9b46ae9e4a0cf6d6782d75_5d916b1a7715625884544afede338b7a is 50, key is test_row_0/A:col10/1732036472627/Put/seqid=0 2024-11-19T17:14:34,134 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d916b1a7715625884544afede338b7a#C#compaction#232 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:34,135 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/8b1cedce661a449fa908a6a685fd67a4 is 50, key is test_row_0/C:col10/1732036471856/Put/seqid=0 2024-11-19T17:14:34,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742098_1274 (size=12454) 2024-11-19T17:14:34,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:34,180 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119824b170d7d9b46ae9e4a0cf6d6782d75_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119824b170d7d9b46ae9e4a0cf6d6782d75_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:34,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742099_1275 (size=13221) 2024-11-19T17:14:34,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/c98ed0703e9e4efd850388ded1441cc0, store: [table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:34,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/c98ed0703e9e4efd850388ded1441cc0 is 175, key is test_row_0/A:col10/1732036472627/Put/seqid=0 2024-11-19T17:14:34,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742100_1276 (size=31255) 2024-11-19T17:14:34,198 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/8b1cedce661a449fa908a6a685fd67a4 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/8b1cedce661a449fa908a6a685fd67a4 2024-11-19T17:14:34,205 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5d916b1a7715625884544afede338b7a/C of 5d916b1a7715625884544afede338b7a into 8b1cedce661a449fa908a6a685fd67a4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:34,205 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:34,205 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., storeName=5d916b1a7715625884544afede338b7a/C, priority=11, startTime=1732036473951; duration=0sec 2024-11-19T17:14:34,205 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:34,205 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d916b1a7715625884544afede338b7a:C 2024-11-19T17:14:34,504 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d916b1a7715625884544afede338b7a#A#compaction#229 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:34,505 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/d78643f1f65a4189bf2b33edfb11840e is 175, key is test_row_0/A:col10/1732036471856/Put/seqid=0 2024-11-19T17:14:34,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742101_1277 (size=32175) 2024-11-19T17:14:34,598 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=509, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/c98ed0703e9e4efd850388ded1441cc0 2024-11-19T17:14:34,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/55b74756f1f941e2a300517204a99568 is 50, key is test_row_0/B:col10/1732036472627/Put/seqid=0 2024-11-19T17:14:34,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742102_1278 (size=12301) 2024-11-19T17:14:34,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:34,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. as already flushing 2024-11-19T17:14:34,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036534815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:34,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:34,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:34,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:34,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036534818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:34,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:34,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:34,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:34,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036534821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:34,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:34,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:34,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:34,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34206 deadline: 1732036534844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:34,846 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_0' in 5d916b1a7715625884544afede338b7a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:14:34,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:34,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:34,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:34,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T17:14:34,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036534924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:34,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:34,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:34,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:34,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:34,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:34,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036534934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:34,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036534934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:34,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:34,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:34,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:34,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:34,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:34,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:34,976 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/d78643f1f65a4189bf2b33edfb11840e as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/d78643f1f65a4189bf2b33edfb11840e 2024-11-19T17:14:34,982 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5d916b1a7715625884544afede338b7a/A of 5d916b1a7715625884544afede338b7a into d78643f1f65a4189bf2b33edfb11840e(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-19T17:14:34,982 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:34,982 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., storeName=5d916b1a7715625884544afede338b7a/A, priority=11, startTime=1732036473951; duration=1sec 2024-11-19T17:14:34,982 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:34,982 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d916b1a7715625884544afede338b7a:A 2024-11-19T17:14:35,075 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/55b74756f1f941e2a300517204a99568 2024-11-19T17:14:35,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/87015f6e98254dbc9cfbb7f8429112d1 is 50, key is test_row_0/C:col10/1732036472627/Put/seqid=0 2024-11-19T17:14:35,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742103_1279 (size=12301) 2024-11-19T17:14:35,091 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/87015f6e98254dbc9cfbb7f8429112d1 2024-11-19T17:14:35,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/c98ed0703e9e4efd850388ded1441cc0 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/c98ed0703e9e4efd850388ded1441cc0 2024-11-19T17:14:35,103 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/c98ed0703e9e4efd850388ded1441cc0, entries=150, sequenceid=509, filesize=30.5 K 2024-11-19T17:14:35,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/55b74756f1f941e2a300517204a99568 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/55b74756f1f941e2a300517204a99568 2024-11-19T17:14:35,108 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/55b74756f1f941e2a300517204a99568, entries=150, sequenceid=509, filesize=12.0 K 2024-11-19T17:14:35,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/87015f6e98254dbc9cfbb7f8429112d1 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/87015f6e98254dbc9cfbb7f8429112d1 2024-11-19T17:14:35,113 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/87015f6e98254dbc9cfbb7f8429112d1, entries=150, sequenceid=509, filesize=12.0 K 2024-11-19T17:14:35,114 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 5d916b1a7715625884544afede338b7a in 1104ms, sequenceid=509, compaction requested=false 2024-11-19T17:14:35,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:35,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:35,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-19T17:14:35,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-19T17:14:35,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-19T17:14:35,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3480 sec 2024-11-19T17:14:35,120 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.3540 sec 2024-11-19T17:14:35,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d916b1a7715625884544afede338b7a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T17:14:35,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=A 2024-11-19T17:14:35,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:35,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=B 2024-11-19T17:14:35,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:35,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=C 2024-11-19T17:14:35,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:35,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192f2eb0924ca5456c8432dbea8594033e_5d916b1a7715625884544afede338b7a is 50, key is test_row_0/A:col10/1732036475135/Put/seqid=0 2024-11-19T17:14:35,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742104_1280 (size=14994) 2024-11-19T17:14:35,159 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:35,165 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192f2eb0924ca5456c8432dbea8594033e_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192f2eb0924ca5456c8432dbea8594033e_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:35,167 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/e6786ab615d8485c89eba0ad3c3adfcf, store: [table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:35,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/e6786ab615d8485c89eba0ad3c3adfcf is 175, key is test_row_0/A:col10/1732036475135/Put/seqid=0 2024-11-19T17:14:35,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:35,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036535187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:35,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732036475169, max timestamp=1732036475192Num uniques 859; 2024-11-19T17:14:35,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T17:14:35,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732036475169, max timestamp=1732036475192Num uniques 859; 2024-11-19T17:14:35,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T17:14:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:35,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732036475169, max timestamp=1732036475192Num uniques 859; 2024-11-19T17:14:35,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T17:14:35,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742105_1281 (size=39949) 2024-11-19T17:14:35,204 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=523, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/e6786ab615d8485c89eba0ad3c3adfcf 2024-11-19T17:14:35,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/9615423c3e3e44a6aa1eaeebceebcf95 is 50, key is test_row_0/B:col10/1732036475135/Put/seqid=0 2024-11-19T17:14:35,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742106_1282 (size=12301) 2024-11-19T17:14:35,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/9615423c3e3e44a6aa1eaeebceebcf95 2024-11-19T17:14:35,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/735ffea78d964408b3b7dfbc6a0c41f5 is 50, key is test_row_0/C:col10/1732036475135/Put/seqid=0 2024-11-19T17:14:35,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:35,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036535238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:35,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:35,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:35,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:35,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742107_1283 (size=12301) 2024-11-19T17:14:35,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:35,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036535244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:35,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:35,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036535303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:35,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:35,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:35,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:35,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:35,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036535354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:35,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:35,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:35,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036535364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:35,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:35,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:35,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:35,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:35,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036535513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:35,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:35,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:35,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:35,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:35,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036535564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:35,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:35,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:35,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:35,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:35,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036535573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:35,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:35,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:35,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:35,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/735ffea78d964408b3b7dfbc6a0c41f5 2024-11-19T17:14:35,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/e6786ab615d8485c89eba0ad3c3adfcf as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/e6786ab615d8485c89eba0ad3c3adfcf 2024-11-19T17:14:35,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/e6786ab615d8485c89eba0ad3c3adfcf, entries=200, sequenceid=523, filesize=39.0 K 2024-11-19T17:14:35,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/9615423c3e3e44a6aa1eaeebceebcf95 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/9615423c3e3e44a6aa1eaeebceebcf95 2024-11-19T17:14:35,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/9615423c3e3e44a6aa1eaeebceebcf95, entries=150, sequenceid=523, filesize=12.0 K 2024-11-19T17:14:35,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/735ffea78d964408b3b7dfbc6a0c41f5 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/735ffea78d964408b3b7dfbc6a0c41f5 2024-11-19T17:14:35,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/735ffea78d964408b3b7dfbc6a0c41f5, entries=150, sequenceid=523, filesize=12.0 K 2024-11-19T17:14:35,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 5d916b1a7715625884544afede338b7a in 532ms, sequenceid=523, compaction requested=true 2024-11-19T17:14:35,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:35,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d916b1a7715625884544afede338b7a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:14:35,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:35,668 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:35,668 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:35,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d916b1a7715625884544afede338b7a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:14:35,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:35,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d916b1a7715625884544afede338b7a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:14:35,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:35,680 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:35,680 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 5d916b1a7715625884544afede338b7a/A is initiating minor compaction (all files) 2024-11-19T17:14:35,681 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d916b1a7715625884544afede338b7a/A in TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:35,681 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/d78643f1f65a4189bf2b33edfb11840e, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/c98ed0703e9e4efd850388ded1441cc0, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/e6786ab615d8485c89eba0ad3c3adfcf] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp, totalSize=101.0 K 2024-11-19T17:14:35,681 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:35,681 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. files: [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/d78643f1f65a4189bf2b33edfb11840e, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/c98ed0703e9e4efd850388ded1441cc0, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/e6786ab615d8485c89eba0ad3c3adfcf] 2024-11-19T17:14:35,682 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:35,682 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 5d916b1a7715625884544afede338b7a/B is initiating minor compaction (all files) 2024-11-19T17:14:35,682 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d916b1a7715625884544afede338b7a/B in TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:35,682 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/1cf96c58b0074a76bac9fa75ae511aa6, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/55b74756f1f941e2a300517204a99568, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/9615423c3e3e44a6aa1eaeebceebcf95] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp, totalSize=36.9 K 2024-11-19T17:14:35,682 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting d78643f1f65a4189bf2b33edfb11840e, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1732036471856 2024-11-19T17:14:35,683 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cf96c58b0074a76bac9fa75ae511aa6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1732036471856 2024-11-19T17:14:35,683 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting c98ed0703e9e4efd850388ded1441cc0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=509, earliestPutTs=1732036472613 2024-11-19T17:14:35,683 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 55b74756f1f941e2a300517204a99568, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=509, earliestPutTs=1732036472613 2024-11-19T17:14:35,683 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6786ab615d8485c89eba0ad3c3adfcf, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1732036474810 2024-11-19T17:14:35,685 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 9615423c3e3e44a6aa1eaeebceebcf95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1732036474810 2024-11-19T17:14:35,699 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:35,711 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d916b1a7715625884544afede338b7a#B#compaction#239 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:35,711 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119eaea1a3d89c94684a046923bc71ccd3c_5d916b1a7715625884544afede338b7a store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:35,711 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/2c2e15ad96914e58a453afc82d627a7a is 50, key is test_row_0/B:col10/1732036475135/Put/seqid=0 2024-11-19T17:14:35,713 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119eaea1a3d89c94684a046923bc71ccd3c_5d916b1a7715625884544afede338b7a, store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:35,713 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119eaea1a3d89c94684a046923bc71ccd3c_5d916b1a7715625884544afede338b7a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:35,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742108_1284 (size=13323) 2024-11-19T17:14:35,724 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/2c2e15ad96914e58a453afc82d627a7a as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/2c2e15ad96914e58a453afc82d627a7a 2024-11-19T17:14:35,733 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d916b1a7715625884544afede338b7a/B of 5d916b1a7715625884544afede338b7a into 2c2e15ad96914e58a453afc82d627a7a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:35,733 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:35,733 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., storeName=5d916b1a7715625884544afede338b7a/B, priority=13, startTime=1732036475668; duration=0sec 2024-11-19T17:14:35,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742109_1285 (size=4469) 2024-11-19T17:14:35,734 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:35,734 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d916b1a7715625884544afede338b7a:B 2024-11-19T17:14:35,734 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:35,736 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:35,736 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d916b1a7715625884544afede338b7a#A#compaction#238 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:35,736 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 5d916b1a7715625884544afede338b7a/C is initiating minor compaction (all files) 2024-11-19T17:14:35,736 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d916b1a7715625884544afede338b7a/C in TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:35,736 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/8b1cedce661a449fa908a6a685fd67a4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/87015f6e98254dbc9cfbb7f8429112d1, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/735ffea78d964408b3b7dfbc6a0c41f5] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp, totalSize=36.9 K 2024-11-19T17:14:35,736 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/3fbdfba3663c48a7a71843016269495a is 175, key is test_row_0/A:col10/1732036475135/Put/seqid=0 2024-11-19T17:14:35,737 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b1cedce661a449fa908a6a685fd67a4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1732036471856 2024-11-19T17:14:35,737 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 87015f6e98254dbc9cfbb7f8429112d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=509, earliestPutTs=1732036472613 2024-11-19T17:14:35,739 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 735ffea78d964408b3b7dfbc6a0c41f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1732036474810 2024-11-19T17:14:35,753 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d916b1a7715625884544afede338b7a#C#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:35,754 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/1cc48f6507774e9a91c6e4e4fd143596 is 50, key is test_row_0/C:col10/1732036475135/Put/seqid=0 2024-11-19T17:14:35,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742110_1286 (size=32277) 2024-11-19T17:14:35,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742111_1287 (size=13323) 2024-11-19T17:14:35,763 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/3fbdfba3663c48a7a71843016269495a as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/3fbdfba3663c48a7a71843016269495a 2024-11-19T17:14:35,770 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d916b1a7715625884544afede338b7a/A of 5d916b1a7715625884544afede338b7a into 3fbdfba3663c48a7a71843016269495a(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:35,770 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:35,770 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., storeName=5d916b1a7715625884544afede338b7a/A, priority=13, startTime=1732036475668; duration=0sec 2024-11-19T17:14:35,770 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:35,770 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d916b1a7715625884544afede338b7a:A 2024-11-19T17:14:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:35,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d916b1a7715625884544afede338b7a 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-19T17:14:35,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=A 2024-11-19T17:14:35,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:35,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=B 2024-11-19T17:14:35,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:35,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=C 2024-11-19T17:14:35,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:35,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190f54c21f1ecf4e8fb2716ae405e64567_5d916b1a7715625884544afede338b7a is 50, key is test_row_0/A:col10/1732036475242/Put/seqid=0 2024-11-19T17:14:35,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742112_1288 (size=14994) 2024-11-19T17:14:35,854 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:35,862 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190f54c21f1ecf4e8fb2716ae405e64567_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190f54c21f1ecf4e8fb2716ae405e64567_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:35,865 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/6e047d07caae4568ac2c199fa1e7fd39, store: [table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:35,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/6e047d07caae4568ac2c199fa1e7fd39 is 175, key is test_row_0/A:col10/1732036475242/Put/seqid=0 2024-11-19T17:14:35,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036535880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:35,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036535884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:35,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:35,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:35,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:35,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:35,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:35,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:35,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036535884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:35,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:35,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:35,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:35,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742113_1289 (size=39949) 2024-11-19T17:14:35,890 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=560, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/6e047d07caae4568ac2c199fa1e7fd39 2024-11-19T17:14:35,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/2a060aa721db42b3a195886abcf83278 is 50, key is test_row_0/B:col10/1732036475242/Put/seqid=0 2024-11-19T17:14:35,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742114_1290 (size=12301) 2024-11-19T17:14:36,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036535993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:36,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:36,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:36,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:36,168 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/1cc48f6507774e9a91c6e4e4fd143596 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/1cc48f6507774e9a91c6e4e4fd143596 2024-11-19T17:14:36,174 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d916b1a7715625884544afede338b7a/C of 5d916b1a7715625884544afede338b7a into 1cc48f6507774e9a91c6e4e4fd143596(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:36,174 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:36,175 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., storeName=5d916b1a7715625884544afede338b7a/C, priority=13, startTime=1732036475668; duration=0sec 2024-11-19T17:14:36,175 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:36,175 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d916b1a7715625884544afede338b7a:C 2024-11-19T17:14:36,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:36,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036536204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:36,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:36,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:36,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:36,303 INFO [Thread-794 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-794. 2024-11-19T17:14:36,303 DEBUG [Thread-794 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:14:36,303 DEBUG [Thread-794 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:36,303 DEBUG [Thread-794 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:36,303 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:14:36,307 INFO [Thread-796 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-796. 2024-11-19T17:14:36,307 DEBUG [Thread-796 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:14:36,307 DEBUG [Thread-796 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:36,307 DEBUG [Thread-796 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:36,307 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:14:36,307 INFO [Thread-792 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-792. 2024-11-19T17:14:36,307 DEBUG [Thread-792 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:14:36,307 DEBUG [Thread-792 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:36,307 DEBUG [Thread-792 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:36,307 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:14:36,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/2a060aa721db42b3a195886abcf83278 2024-11-19T17:14:36,310 INFO [Thread-798 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-798. 2024-11-19T17:14:36,310 DEBUG [Thread-798 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:14:36,310 DEBUG [Thread-798 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:36,310 DEBUG [Thread-798 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:36,311 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:14:36,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/fbad9f5a394d4f9186e3679e00c0c2a0 is 50, key is test_row_0/C:col10/1732036475242/Put/seqid=0 2024-11-19T17:14:36,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742115_1291 (size=12301) 2024-11-19T17:14:36,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:36,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:36,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34158 deadline: 1732036536394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:36,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34196 deadline: 1732036536394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:36,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:36,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:36,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:36,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:36,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:36,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:36,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34212 deadline: 1732036536514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:36,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:36,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:36,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:14:36,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/fbad9f5a394d4f9186e3679e00c0c2a0 2024-11-19T17:14:36,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/6e047d07caae4568ac2c199fa1e7fd39 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/6e047d07caae4568ac2c199fa1e7fd39 2024-11-19T17:14:36,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/6e047d07caae4568ac2c199fa1e7fd39, entries=200, sequenceid=560, filesize=39.0 K 2024-11-19T17:14:36,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/2a060aa721db42b3a195886abcf83278 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/2a060aa721db42b3a195886abcf83278 2024-11-19T17:14:36,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/2a060aa721db42b3a195886abcf83278, entries=150, sequenceid=560, filesize=12.0 K 2024-11-19T17:14:36,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/fbad9f5a394d4f9186e3679e00c0c2a0 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/fbad9f5a394d4f9186e3679e00c0c2a0 2024-11-19T17:14:36,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/fbad9f5a394d4f9186e3679e00c0c2a0, entries=150, sequenceid=560, filesize=12.0 K 2024-11-19T17:14:36,742 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 5d916b1a7715625884544afede338b7a in 909ms, sequenceid=560, compaction requested=false 2024-11-19T17:14:36,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:36,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:36,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d916b1a7715625884544afede338b7a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T17:14:36,876 INFO [Thread-789 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-789. 2024-11-19T17:14:36,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=A 2024-11-19T17:14:36,876 DEBUG [Thread-789 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:14:36,876 DEBUG [Thread-789 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:36,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:36,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=B 2024-11-19T17:14:36,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:36,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=C 2024-11-19T17:14:36,876 DEBUG [Thread-789 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:36,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:36,876 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:14:36,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119051b1d1e99bf4d708b50cfb0f3a99e5e_5d916b1a7715625884544afede338b7a is 50, key is test_row_0/A:col10/1732036476874/Put/seqid=0 2024-11-19T17:14:36,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742116_1292 (size=9914) 2024-11-19T17:14:36,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T17:14:36,944 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:14:37,026 INFO [Thread-781 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-781. 2024-11-19T17:14:37,026 DEBUG [Thread-781 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:14:37,026 DEBUG [Thread-781 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:37,026 DEBUG [Thread-781 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:37,026 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:14:37,286 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:37,290 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119051b1d1e99bf4d708b50cfb0f3a99e5e_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119051b1d1e99bf4d708b50cfb0f3a99e5e_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:37,290 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/d6b56ab665de43a8a039e8e5cf60c0e7, store: [table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:37,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/d6b56ab665de43a8a039e8e5cf60c0e7 is 175, key is test_row_0/A:col10/1732036476874/Put/seqid=0 2024-11-19T17:14:37,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742117_1293 (size=22561) 2024-11-19T17:14:37,405 INFO [Thread-783 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-783. 2024-11-19T17:14:37,406 DEBUG [Thread-783 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:14:37,406 DEBUG [Thread-783 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:37,406 DEBUG [Thread-783 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:37,406 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:14:37,416 INFO [Thread-787 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-787. 2024-11-19T17:14:37,416 DEBUG [Thread-787 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:14:37,416 DEBUG [Thread-787 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:37,416 DEBUG [Thread-787 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:37,416 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:14:37,695 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=572, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/d6b56ab665de43a8a039e8e5cf60c0e7 2024-11-19T17:14:37,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/506f995a475d44e08edf3c9ac865a8e3 is 50, key is test_row_0/B:col10/1732036476874/Put/seqid=0 2024-11-19T17:14:37,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742118_1294 (size=9857) 2024-11-19T17:14:38,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/506f995a475d44e08edf3c9ac865a8e3 2024-11-19T17:14:38,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/3a8f8850f2a24bc7bff3c8432ea700d2 is 50, key is test_row_0/C:col10/1732036476874/Put/seqid=0 2024-11-19T17:14:38,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742119_1295 (size=9857) 2024-11-19T17:14:38,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/3a8f8850f2a24bc7bff3c8432ea700d2 2024-11-19T17:14:38,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/d6b56ab665de43a8a039e8e5cf60c0e7 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/d6b56ab665de43a8a039e8e5cf60c0e7 2024-11-19T17:14:38,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/d6b56ab665de43a8a039e8e5cf60c0e7, entries=100, sequenceid=572, filesize=22.0 K 2024-11-19T17:14:38,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/506f995a475d44e08edf3c9ac865a8e3 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/506f995a475d44e08edf3c9ac865a8e3 2024-11-19T17:14:38,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/506f995a475d44e08edf3c9ac865a8e3, entries=100, sequenceid=572, filesize=9.6 K 2024-11-19T17:14:38,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/3a8f8850f2a24bc7bff3c8432ea700d2 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/3a8f8850f2a24bc7bff3c8432ea700d2 2024-11-19T17:14:38,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/3a8f8850f2a24bc7bff3c8432ea700d2, entries=100, sequenceid=572, filesize=9.6 K 2024-11-19T17:14:38,562 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=20.13 KB/20610 for 5d916b1a7715625884544afede338b7a in 1687ms, sequenceid=572, compaction requested=true 2024-11-19T17:14:38,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:38,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d916b1a7715625884544afede338b7a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:14:38,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:38,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d916b1a7715625884544afede338b7a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:14:38,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:38,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d916b1a7715625884544afede338b7a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:14:38,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:38,562 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:38,562 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:38,564 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94787 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:38,564 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 5d916b1a7715625884544afede338b7a/A is initiating minor compaction (all files) 2024-11-19T17:14:38,564 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d916b1a7715625884544afede338b7a/A in TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:38,564 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:38,564 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 5d916b1a7715625884544afede338b7a/B is initiating minor compaction (all files) 2024-11-19T17:14:38,564 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/3fbdfba3663c48a7a71843016269495a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/6e047d07caae4568ac2c199fa1e7fd39, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/d6b56ab665de43a8a039e8e5cf60c0e7] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp, totalSize=92.6 K 2024-11-19T17:14:38,564 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d916b1a7715625884544afede338b7a/B in TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:38,564 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:38,564 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/2c2e15ad96914e58a453afc82d627a7a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/2a060aa721db42b3a195886abcf83278, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/506f995a475d44e08edf3c9ac865a8e3] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp, totalSize=34.6 K 2024-11-19T17:14:38,564 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. files: [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/3fbdfba3663c48a7a71843016269495a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/6e047d07caae4568ac2c199fa1e7fd39, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/d6b56ab665de43a8a039e8e5cf60c0e7] 2024-11-19T17:14:38,565 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c2e15ad96914e58a453afc82d627a7a, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1732036474810 2024-11-19T17:14:38,565 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fbdfba3663c48a7a71843016269495a, keycount=150, bloomtype=ROW, size=31.5 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1732036474810 2024-11-19T17:14:38,566 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e047d07caae4568ac2c199fa1e7fd39, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=560, earliestPutTs=1732036475222 2024-11-19T17:14:38,567 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a060aa721db42b3a195886abcf83278, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=560, earliestPutTs=1732036475236 2024-11-19T17:14:38,567 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6b56ab665de43a8a039e8e5cf60c0e7, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1732036475876 2024-11-19T17:14:38,567 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 506f995a475d44e08edf3c9ac865a8e3, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1732036475876 2024-11-19T17:14:38,579 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d916b1a7715625884544afede338b7a#B#compaction#247 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:38,579 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:38,579 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/d98ef386d5b740e4989635bf913015a8 is 50, key is test_row_0/B:col10/1732036476874/Put/seqid=0 2024-11-19T17:14:38,588 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411195c92ead3a2124009b3dde5c963040a1d_5d916b1a7715625884544afede338b7a store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:38,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742120_1296 (size=13425) 2024-11-19T17:14:38,607 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411195c92ead3a2124009b3dde5c963040a1d_5d916b1a7715625884544afede338b7a, store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:38,607 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195c92ead3a2124009b3dde5c963040a1d_5d916b1a7715625884544afede338b7a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:38,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742121_1297 (size=4469) 2024-11-19T17:14:38,997 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/d98ef386d5b740e4989635bf913015a8 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/d98ef386d5b740e4989635bf913015a8 2024-11-19T17:14:39,002 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d916b1a7715625884544afede338b7a/B of 5d916b1a7715625884544afede338b7a into d98ef386d5b740e4989635bf913015a8(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:39,002 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:39,002 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., storeName=5d916b1a7715625884544afede338b7a/B, priority=13, startTime=1732036478562; duration=0sec 2024-11-19T17:14:39,002 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:39,002 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d916b1a7715625884544afede338b7a:B 2024-11-19T17:14:39,002 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:39,003 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:39,003 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 5d916b1a7715625884544afede338b7a/C is initiating minor compaction (all files) 2024-11-19T17:14:39,003 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d916b1a7715625884544afede338b7a/C in TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:39,003 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/1cc48f6507774e9a91c6e4e4fd143596, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/fbad9f5a394d4f9186e3679e00c0c2a0, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/3a8f8850f2a24bc7bff3c8432ea700d2] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp, totalSize=34.6 K 2024-11-19T17:14:39,003 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cc48f6507774e9a91c6e4e4fd143596, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1732036474810 2024-11-19T17:14:39,004 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting fbad9f5a394d4f9186e3679e00c0c2a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=560, earliestPutTs=1732036475236 2024-11-19T17:14:39,004 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a8f8850f2a24bc7bff3c8432ea700d2, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1732036475876 2024-11-19T17:14:39,012 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d916b1a7715625884544afede338b7a#C#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:39,012 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/9ef4a6327e2a4b26aa6f01e8bc043d69 is 50, key is test_row_0/C:col10/1732036476874/Put/seqid=0 2024-11-19T17:14:39,013 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d916b1a7715625884544afede338b7a#A#compaction#248 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:39,013 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/81ceb0cc2b9e493abf821017690a071d is 175, key is test_row_0/A:col10/1732036476874/Put/seqid=0 2024-11-19T17:14:39,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742122_1298 (size=13425) 2024-11-19T17:14:39,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742123_1299 (size=32486) 2024-11-19T17:14:39,422 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/9ef4a6327e2a4b26aa6f01e8bc043d69 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/9ef4a6327e2a4b26aa6f01e8bc043d69 2024-11-19T17:14:39,427 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d916b1a7715625884544afede338b7a/C of 5d916b1a7715625884544afede338b7a into 9ef4a6327e2a4b26aa6f01e8bc043d69(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:39,427 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:39,427 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., storeName=5d916b1a7715625884544afede338b7a/C, priority=13, startTime=1732036478562; duration=0sec 2024-11-19T17:14:39,427 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:39,427 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d916b1a7715625884544afede338b7a:C 2024-11-19T17:14:39,428 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/81ceb0cc2b9e493abf821017690a071d as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/81ceb0cc2b9e493abf821017690a071d 2024-11-19T17:14:39,433 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d916b1a7715625884544afede338b7a/A of 5d916b1a7715625884544afede338b7a into 81ceb0cc2b9e493abf821017690a071d(size=31.7 K), total size for store is 31.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:39,433 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d916b1a7715625884544afede338b7a: 2024-11-19T17:14:39,433 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., storeName=5d916b1a7715625884544afede338b7a/A, priority=13, startTime=1732036478562; duration=0sec 2024-11-19T17:14:39,433 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:39,433 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d916b1a7715625884544afede338b7a:A 2024-11-19T17:14:40,709 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T17:14:43,916 INFO [Thread-785 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-785. 2024-11-19T17:14:43,916 DEBUG [Thread-785 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:14:43,916 DEBUG [Thread-785 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:43,916 DEBUG [Thread-785 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:43,916 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:14:43,916 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T17:14:43,916 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-11-19T17:14:43,916 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 113 2024-11-19T17:14:43,916 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-19T17:14:43,916 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 119 2024-11-19T17:14:43,916 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-19T17:14:43,916 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T17:14:43,916 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2220 2024-11-19T17:14:43,916 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2275 2024-11-19T17:14:43,916 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T17:14:43,916 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1031 2024-11-19T17:14:43,916 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3093 rows 2024-11-19T17:14:43,916 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1033 2024-11-19T17:14:43,916 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3095 rows 2024-11-19T17:14:43,917 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T17:14:43,917 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T17:14:43,917 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:43,917 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:43,917 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:14:43,918 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T17:14:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T17:14:43,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T17:14:43,923 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732036483923"}]},"ts":"1732036483923"} 2024-11-19T17:14:43,925 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T17:14:43,925 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T17:14:43,926 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T17:14:43,928 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d916b1a7715625884544afede338b7a, UNASSIGN}] 2024-11-19T17:14:43,929 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d916b1a7715625884544afede338b7a, UNASSIGN 2024-11-19T17:14:43,930 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=5d916b1a7715625884544afede338b7a, regionState=CLOSING, regionLocation=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:43,931 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d916b1a7715625884544afede338b7a, UNASSIGN because future has completed 2024-11-19T17:14:43,932 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T17:14:43,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473}] 2024-11-19T17:14:44,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T17:14:44,085 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] handler.UnassignRegionHandler(122): Close 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:44,085 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T17:14:44,085 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegion(1722): Closing 5d916b1a7715625884544afede338b7a, disabling compactions & flushes 2024-11-19T17:14:44,085 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:44,085 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:44,085 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. after waiting 0 ms 2024-11-19T17:14:44,085 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:44,085 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegion(2902): Flushing 5d916b1a7715625884544afede338b7a 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-19T17:14:44,086 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=A 2024-11-19T17:14:44,086 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:44,086 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=B 2024-11-19T17:14:44,086 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:44,086 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5d916b1a7715625884544afede338b7a, store=C 2024-11-19T17:14:44,086 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:44,093 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198d0527a55cda4629b9321109b51b7549_5d916b1a7715625884544afede338b7a is 50, key is test_row_0/A:col10/1732036483914/Put/seqid=0 2024-11-19T17:14:44,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742124_1300 (size=12454) 2024-11-19T17:14:44,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T17:14:44,499 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:44,503 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198d0527a55cda4629b9321109b51b7549_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198d0527a55cda4629b9321109b51b7549_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:44,504 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/c3c87d3d70db4adab663ec7bfaee8b2c, store: [table=TestAcidGuarantees family=A region=5d916b1a7715625884544afede338b7a] 2024-11-19T17:14:44,504 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/c3c87d3d70db4adab663ec7bfaee8b2c is 175, key is test_row_0/A:col10/1732036483914/Put/seqid=0 2024-11-19T17:14:44,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742125_1301 (size=31255) 2024-11-19T17:14:44,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T17:14:44,908 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=582, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/c3c87d3d70db4adab663ec7bfaee8b2c 2024-11-19T17:14:44,915 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/ec8d7779d626473f91dea7795948450d is 50, key is test_row_0/B:col10/1732036483914/Put/seqid=0 2024-11-19T17:14:44,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742126_1302 (size=12301) 2024-11-19T17:14:45,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T17:14:45,319 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=582 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/ec8d7779d626473f91dea7795948450d 2024-11-19T17:14:45,327 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/5189713c51024576b96cb42f5fa47823 is 50, key is test_row_0/C:col10/1732036483914/Put/seqid=0 2024-11-19T17:14:45,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742127_1303 (size=12301) 2024-11-19T17:14:45,335 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=582 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/5189713c51024576b96cb42f5fa47823 2024-11-19T17:14:45,339 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/A/c3c87d3d70db4adab663ec7bfaee8b2c as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/c3c87d3d70db4adab663ec7bfaee8b2c 2024-11-19T17:14:45,343 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/c3c87d3d70db4adab663ec7bfaee8b2c, entries=150, sequenceid=582, filesize=30.5 K 2024-11-19T17:14:45,343 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/B/ec8d7779d626473f91dea7795948450d as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/ec8d7779d626473f91dea7795948450d 2024-11-19T17:14:45,347 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/ec8d7779d626473f91dea7795948450d, entries=150, sequenceid=582, filesize=12.0 K 2024-11-19T17:14:45,348 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/.tmp/C/5189713c51024576b96cb42f5fa47823 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/5189713c51024576b96cb42f5fa47823 2024-11-19T17:14:45,351 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/5189713c51024576b96cb42f5fa47823, entries=150, sequenceid=582, filesize=12.0 K 2024-11-19T17:14:45,352 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 5d916b1a7715625884544afede338b7a in 1267ms, sequenceid=582, compaction requested=false 2024-11-19T17:14:45,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/f80dd0aeeb17414db815143a1979b3a0, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/6026ce3e3f3a4dd28e1f25e803c3db6d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/ec74e8e9e56a47ed8e736ed046620633, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/97ce65aa269a4823936ed378f4482192, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/5447bf1cd9c04c9f9ac82074364bf762, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/b862fad5d9954e35a74e2cd634ec5035, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/e1de6532dbe44dbcaffa90b4c68abe39, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/4185312748934eeab86c30030e714504, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/6393eaf22bdb459b9eda4e34caf23f35, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/a9b82d2de3e14e208789283566038e2d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/ef45c671b8c54f69b17ef2b613b722db, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/fc4c03a222ea434991a1ab703c95f7fb, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/de841b53fbcc4059a4c71ccb4c05c15c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/24b583e3f5fc496e9045baa84202763d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/4d7bd0e29f4c4004bb2a120f0471eafb, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/e17196e1c4d941e89b4cc6fc998a328b, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/80ce05b478cf452d8586a27015d8f2bf, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/ed214672a3774e048dce0f4e6c8c2315, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/be74860499714bb2947b347fbc3017e1, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/d7d94027d1604df58b35a5c9f59d08e1, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/8ea4eb0d84e945b991123934410babbc, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/4f549abb53ee49a0895a6cb520f87f1b, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/d24b96ac00de49e8ac47feff9ce0fb3d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/38ba54e3a0f14202bbb7dafb289beed4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/7639f21e5d574b278daa1f1ae6bc5440, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/05aabb1b3bca449388e24d29301e04a9, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/6cb4d2f0ff994c4e8d45b20e402bf634, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/d78643f1f65a4189bf2b33edfb11840e, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/c98ed0703e9e4efd850388ded1441cc0, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/e6786ab615d8485c89eba0ad3c3adfcf, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/3fbdfba3663c48a7a71843016269495a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/6e047d07caae4568ac2c199fa1e7fd39, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/d6b56ab665de43a8a039e8e5cf60c0e7] to archive 2024-11-19T17:14:45,354 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:14:45,356 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/f80dd0aeeb17414db815143a1979b3a0 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/f80dd0aeeb17414db815143a1979b3a0 2024-11-19T17:14:45,357 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/6026ce3e3f3a4dd28e1f25e803c3db6d to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/6026ce3e3f3a4dd28e1f25e803c3db6d 2024-11-19T17:14:45,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/ec74e8e9e56a47ed8e736ed046620633 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/ec74e8e9e56a47ed8e736ed046620633 2024-11-19T17:14:45,359 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/97ce65aa269a4823936ed378f4482192 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/97ce65aa269a4823936ed378f4482192 2024-11-19T17:14:45,359 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/5447bf1cd9c04c9f9ac82074364bf762 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/5447bf1cd9c04c9f9ac82074364bf762 2024-11-19T17:14:45,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/b862fad5d9954e35a74e2cd634ec5035 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/b862fad5d9954e35a74e2cd634ec5035 2024-11-19T17:14:45,362 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/e1de6532dbe44dbcaffa90b4c68abe39 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/e1de6532dbe44dbcaffa90b4c68abe39 2024-11-19T17:14:45,363 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/4185312748934eeab86c30030e714504 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/4185312748934eeab86c30030e714504 2024-11-19T17:14:45,364 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/6393eaf22bdb459b9eda4e34caf23f35 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/6393eaf22bdb459b9eda4e34caf23f35 2024-11-19T17:14:45,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/a9b82d2de3e14e208789283566038e2d to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/a9b82d2de3e14e208789283566038e2d 2024-11-19T17:14:45,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/ef45c671b8c54f69b17ef2b613b722db to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/ef45c671b8c54f69b17ef2b613b722db 2024-11-19T17:14:45,368 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/fc4c03a222ea434991a1ab703c95f7fb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/fc4c03a222ea434991a1ab703c95f7fb 2024-11-19T17:14:45,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/de841b53fbcc4059a4c71ccb4c05c15c to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/de841b53fbcc4059a4c71ccb4c05c15c 2024-11-19T17:14:45,370 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/24b583e3f5fc496e9045baa84202763d to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/24b583e3f5fc496e9045baa84202763d 2024-11-19T17:14:45,371 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/4d7bd0e29f4c4004bb2a120f0471eafb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/4d7bd0e29f4c4004bb2a120f0471eafb 2024-11-19T17:14:45,372 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/e17196e1c4d941e89b4cc6fc998a328b to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/e17196e1c4d941e89b4cc6fc998a328b 2024-11-19T17:14:45,373 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/80ce05b478cf452d8586a27015d8f2bf to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/80ce05b478cf452d8586a27015d8f2bf 2024-11-19T17:14:45,374 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/ed214672a3774e048dce0f4e6c8c2315 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/ed214672a3774e048dce0f4e6c8c2315 2024-11-19T17:14:45,376 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/be74860499714bb2947b347fbc3017e1 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/be74860499714bb2947b347fbc3017e1 2024-11-19T17:14:45,377 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/d7d94027d1604df58b35a5c9f59d08e1 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/d7d94027d1604df58b35a5c9f59d08e1 2024-11-19T17:14:45,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/8ea4eb0d84e945b991123934410babbc to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/8ea4eb0d84e945b991123934410babbc 2024-11-19T17:14:45,379 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/4f549abb53ee49a0895a6cb520f87f1b to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/4f549abb53ee49a0895a6cb520f87f1b 2024-11-19T17:14:45,380 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/d24b96ac00de49e8ac47feff9ce0fb3d to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/d24b96ac00de49e8ac47feff9ce0fb3d 2024-11-19T17:14:45,381 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/38ba54e3a0f14202bbb7dafb289beed4 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/38ba54e3a0f14202bbb7dafb289beed4 2024-11-19T17:14:45,382 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/7639f21e5d574b278daa1f1ae6bc5440 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/7639f21e5d574b278daa1f1ae6bc5440 2024-11-19T17:14:45,383 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/05aabb1b3bca449388e24d29301e04a9 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/05aabb1b3bca449388e24d29301e04a9 2024-11-19T17:14:45,384 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/6cb4d2f0ff994c4e8d45b20e402bf634 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/6cb4d2f0ff994c4e8d45b20e402bf634 2024-11-19T17:14:45,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/d78643f1f65a4189bf2b33edfb11840e to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/d78643f1f65a4189bf2b33edfb11840e 2024-11-19T17:14:45,387 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/c98ed0703e9e4efd850388ded1441cc0 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/c98ed0703e9e4efd850388ded1441cc0 2024-11-19T17:14:45,388 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/e6786ab615d8485c89eba0ad3c3adfcf to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/e6786ab615d8485c89eba0ad3c3adfcf 2024-11-19T17:14:45,389 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/3fbdfba3663c48a7a71843016269495a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/3fbdfba3663c48a7a71843016269495a 2024-11-19T17:14:45,390 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/6e047d07caae4568ac2c199fa1e7fd39 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/6e047d07caae4568ac2c199fa1e7fd39 2024-11-19T17:14:45,391 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/d6b56ab665de43a8a039e8e5cf60c0e7 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/d6b56ab665de43a8a039e8e5cf60c0e7 2024-11-19T17:14:45,392 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/5ad25038d8974686b2c4c71a087382f2, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/58b2dfc76cc3431fad0db6e68eb0b706, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/b599280704774080bf46549ff45c5889, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/27061b2342ca45bba50b2ab40d5a2614, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/0fc07bf6a56a46658c25d7e8e42c18a7, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/a3c97a7486ce4117af6b1f6b2580f379, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/20589b3e89654314a85dafdc8d619236, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/d08250d2e2e74f0cbcfba45d45bc25d0, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/c42291dbb2bc44ec850bc3e333b44858, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/beb994ac92c5473db2e0d995ece88e11, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/cef3e4d46f18423f84f151e21e867bca, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/68ee75dbbf984b468902a4599e49071a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/08b531d2997f42fd8b7e421eda7e7457, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/74d6483b2b824c53a083b27e44dedecb, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/4ad2ae9bb2af43efa9c49fb508283bae, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/64e2d1bf96f047f38da96274db5624ae, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/038f89d119c143ee9921a205831ec2ca, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/946da17212fa4adb8844b9f6e72179ea, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/c3785d54940f46059519be9d14228608, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/429c3ebc9eb647d7b819b4e52d64efdc, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/5933c0f2cdd04da3860452449b3e1d66, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/e8b8e689a7224deaa457a1d96bece33e, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/7940786ceda74b02b08ea0941e73f2b3, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/67e5ace200564580a1ddf8fdc5884f10, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/0e3a7e9baa9441f18cd60a9efce910dd, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/e61dcf9e2c9841f383db46e057322863, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/1cf96c58b0074a76bac9fa75ae511aa6, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/52a19c7046034776a8af5330060a6bec, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/55b74756f1f941e2a300517204a99568, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/2c2e15ad96914e58a453afc82d627a7a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/9615423c3e3e44a6aa1eaeebceebcf95, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/2a060aa721db42b3a195886abcf83278, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/506f995a475d44e08edf3c9ac865a8e3] to archive 2024-11-19T17:14:45,393 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:14:45,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/5ad25038d8974686b2c4c71a087382f2 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/5ad25038d8974686b2c4c71a087382f2 2024-11-19T17:14:45,395 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/58b2dfc76cc3431fad0db6e68eb0b706 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/58b2dfc76cc3431fad0db6e68eb0b706 2024-11-19T17:14:45,396 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/b599280704774080bf46549ff45c5889 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/b599280704774080bf46549ff45c5889 2024-11-19T17:14:45,397 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/27061b2342ca45bba50b2ab40d5a2614 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/27061b2342ca45bba50b2ab40d5a2614 2024-11-19T17:14:45,398 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/0fc07bf6a56a46658c25d7e8e42c18a7 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/0fc07bf6a56a46658c25d7e8e42c18a7 2024-11-19T17:14:45,398 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/a3c97a7486ce4117af6b1f6b2580f379 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/a3c97a7486ce4117af6b1f6b2580f379 2024-11-19T17:14:45,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/20589b3e89654314a85dafdc8d619236 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/20589b3e89654314a85dafdc8d619236 2024-11-19T17:14:45,400 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/d08250d2e2e74f0cbcfba45d45bc25d0 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/d08250d2e2e74f0cbcfba45d45bc25d0 2024-11-19T17:14:45,401 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/c42291dbb2bc44ec850bc3e333b44858 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/c42291dbb2bc44ec850bc3e333b44858 2024-11-19T17:14:45,403 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/beb994ac92c5473db2e0d995ece88e11 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/beb994ac92c5473db2e0d995ece88e11 2024-11-19T17:14:45,404 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/cef3e4d46f18423f84f151e21e867bca to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/cef3e4d46f18423f84f151e21e867bca 2024-11-19T17:14:45,405 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/68ee75dbbf984b468902a4599e49071a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/68ee75dbbf984b468902a4599e49071a 2024-11-19T17:14:45,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/08b531d2997f42fd8b7e421eda7e7457 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/08b531d2997f42fd8b7e421eda7e7457 2024-11-19T17:14:45,407 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/74d6483b2b824c53a083b27e44dedecb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/74d6483b2b824c53a083b27e44dedecb 2024-11-19T17:14:45,408 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/4ad2ae9bb2af43efa9c49fb508283bae to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/4ad2ae9bb2af43efa9c49fb508283bae 2024-11-19T17:14:45,410 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/64e2d1bf96f047f38da96274db5624ae to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/64e2d1bf96f047f38da96274db5624ae 2024-11-19T17:14:45,411 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/038f89d119c143ee9921a205831ec2ca to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/038f89d119c143ee9921a205831ec2ca 2024-11-19T17:14:45,412 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/946da17212fa4adb8844b9f6e72179ea to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/946da17212fa4adb8844b9f6e72179ea 2024-11-19T17:14:45,413 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/c3785d54940f46059519be9d14228608 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/c3785d54940f46059519be9d14228608 2024-11-19T17:14:45,414 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/429c3ebc9eb647d7b819b4e52d64efdc to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/429c3ebc9eb647d7b819b4e52d64efdc 2024-11-19T17:14:45,416 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/5933c0f2cdd04da3860452449b3e1d66 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/5933c0f2cdd04da3860452449b3e1d66 2024-11-19T17:14:45,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/e8b8e689a7224deaa457a1d96bece33e to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/e8b8e689a7224deaa457a1d96bece33e 2024-11-19T17:14:45,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/7940786ceda74b02b08ea0941e73f2b3 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/7940786ceda74b02b08ea0941e73f2b3 2024-11-19T17:14:45,419 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/67e5ace200564580a1ddf8fdc5884f10 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/67e5ace200564580a1ddf8fdc5884f10 2024-11-19T17:14:45,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/0e3a7e9baa9441f18cd60a9efce910dd to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/0e3a7e9baa9441f18cd60a9efce910dd 2024-11-19T17:14:45,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/e61dcf9e2c9841f383db46e057322863 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/e61dcf9e2c9841f383db46e057322863 2024-11-19T17:14:45,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/1cf96c58b0074a76bac9fa75ae511aa6 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/1cf96c58b0074a76bac9fa75ae511aa6 2024-11-19T17:14:45,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/52a19c7046034776a8af5330060a6bec to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/52a19c7046034776a8af5330060a6bec 2024-11-19T17:14:45,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/55b74756f1f941e2a300517204a99568 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/55b74756f1f941e2a300517204a99568 2024-11-19T17:14:45,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/2c2e15ad96914e58a453afc82d627a7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/2c2e15ad96914e58a453afc82d627a7a 2024-11-19T17:14:45,427 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/9615423c3e3e44a6aa1eaeebceebcf95 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/9615423c3e3e44a6aa1eaeebceebcf95 2024-11-19T17:14:45,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/2a060aa721db42b3a195886abcf83278 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/2a060aa721db42b3a195886abcf83278 2024-11-19T17:14:45,429 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/506f995a475d44e08edf3c9ac865a8e3 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/506f995a475d44e08edf3c9ac865a8e3 2024-11-19T17:14:45,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/b449ac56500d42b890de2fa65c9d76e8, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/a7fbebc0be6842a78026dbe20a32a075, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/923dea8eb3e348e59cbd56a21d29260f, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/c0123e5e1d50477b81bc76d0b5f3b2e5, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/ec84485fab9d4f26ac59e9bcc869427a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/b77d7a72cb7b4b61a593d31ae8b9558e, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/9b12eb42d87544bd8f77b1bfecac36c3, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/c303399ac8f34915ad70e60d7ef4e6cc, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/ec872f2e4fd84e1590fc0a285e59e71b, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/0dc178fada1240e8b0e0b0bbde20a8da, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/f2a3671ca1b34cecbe848b2228ee3414, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/b710f93f0b094d829b5702ec9e4045a5, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/5d0c8452d3004ce6a14b5565f5be2386, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/21929e2424c74fe684ab756f47962a10, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/a2767f2e1ec9431d9ac4fe2a0ce578fe, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/4168946b95ca46319c589f7e6fafe7b5, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/955e797a398043fdac9cd10e73b195d2, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/08dfd8a435764d06b911e84d05619056, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/55aa9e1f05b34f4e81397bb5f2f8f22a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/1f66f7766abd495a9578c060ca229785, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/7d7c67bbc01c4400a3587c9899558bf7, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/e84243131e9a48268f6ce7edd8ed8ff3, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/2c71e965a752452384fafb7f3b4d8c78, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/6193bd9952404e0b81635fce6460f1d5, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/1e1551c8e0384da697ef1fd047bb4de4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/c9ecb62cb8f2440a8cbce2ecca81b1d5, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/8b1cedce661a449fa908a6a685fd67a4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/dfe2722f821d4d63860f170c16b160ff, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/87015f6e98254dbc9cfbb7f8429112d1, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/1cc48f6507774e9a91c6e4e4fd143596, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/735ffea78d964408b3b7dfbc6a0c41f5, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/fbad9f5a394d4f9186e3679e00c0c2a0, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/3a8f8850f2a24bc7bff3c8432ea700d2] to archive 2024-11-19T17:14:45,431 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:14:45,433 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/b449ac56500d42b890de2fa65c9d76e8 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/b449ac56500d42b890de2fa65c9d76e8 2024-11-19T17:14:45,434 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/a7fbebc0be6842a78026dbe20a32a075 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/a7fbebc0be6842a78026dbe20a32a075 2024-11-19T17:14:45,435 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/923dea8eb3e348e59cbd56a21d29260f to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/923dea8eb3e348e59cbd56a21d29260f 2024-11-19T17:14:45,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/c0123e5e1d50477b81bc76d0b5f3b2e5 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/c0123e5e1d50477b81bc76d0b5f3b2e5 2024-11-19T17:14:45,437 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/ec84485fab9d4f26ac59e9bcc869427a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/ec84485fab9d4f26ac59e9bcc869427a 2024-11-19T17:14:45,438 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/b77d7a72cb7b4b61a593d31ae8b9558e to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/b77d7a72cb7b4b61a593d31ae8b9558e 2024-11-19T17:14:45,439 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/9b12eb42d87544bd8f77b1bfecac36c3 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/9b12eb42d87544bd8f77b1bfecac36c3 2024-11-19T17:14:45,440 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/c303399ac8f34915ad70e60d7ef4e6cc to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/c303399ac8f34915ad70e60d7ef4e6cc 2024-11-19T17:14:45,441 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/ec872f2e4fd84e1590fc0a285e59e71b to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/ec872f2e4fd84e1590fc0a285e59e71b 2024-11-19T17:14:45,442 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/0dc178fada1240e8b0e0b0bbde20a8da to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/0dc178fada1240e8b0e0b0bbde20a8da 2024-11-19T17:14:45,443 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/f2a3671ca1b34cecbe848b2228ee3414 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/f2a3671ca1b34cecbe848b2228ee3414 2024-11-19T17:14:45,444 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/b710f93f0b094d829b5702ec9e4045a5 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/b710f93f0b094d829b5702ec9e4045a5 2024-11-19T17:14:45,445 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/5d0c8452d3004ce6a14b5565f5be2386 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/5d0c8452d3004ce6a14b5565f5be2386 2024-11-19T17:14:45,446 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/21929e2424c74fe684ab756f47962a10 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/21929e2424c74fe684ab756f47962a10 2024-11-19T17:14:45,447 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/a2767f2e1ec9431d9ac4fe2a0ce578fe to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/a2767f2e1ec9431d9ac4fe2a0ce578fe 2024-11-19T17:14:45,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/4168946b95ca46319c589f7e6fafe7b5 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/4168946b95ca46319c589f7e6fafe7b5 2024-11-19T17:14:45,449 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/955e797a398043fdac9cd10e73b195d2 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/955e797a398043fdac9cd10e73b195d2 2024-11-19T17:14:45,450 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/08dfd8a435764d06b911e84d05619056 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/08dfd8a435764d06b911e84d05619056 2024-11-19T17:14:45,452 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/55aa9e1f05b34f4e81397bb5f2f8f22a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/55aa9e1f05b34f4e81397bb5f2f8f22a 2024-11-19T17:14:45,453 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/1f66f7766abd495a9578c060ca229785 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/1f66f7766abd495a9578c060ca229785 2024-11-19T17:14:45,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/7d7c67bbc01c4400a3587c9899558bf7 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/7d7c67bbc01c4400a3587c9899558bf7 2024-11-19T17:14:45,455 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/e84243131e9a48268f6ce7edd8ed8ff3 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/e84243131e9a48268f6ce7edd8ed8ff3 2024-11-19T17:14:45,456 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/2c71e965a752452384fafb7f3b4d8c78 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/2c71e965a752452384fafb7f3b4d8c78 2024-11-19T17:14:45,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/6193bd9952404e0b81635fce6460f1d5 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/6193bd9952404e0b81635fce6460f1d5 2024-11-19T17:14:45,458 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/1e1551c8e0384da697ef1fd047bb4de4 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/1e1551c8e0384da697ef1fd047bb4de4 2024-11-19T17:14:45,459 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/c9ecb62cb8f2440a8cbce2ecca81b1d5 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/c9ecb62cb8f2440a8cbce2ecca81b1d5 2024-11-19T17:14:45,460 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/8b1cedce661a449fa908a6a685fd67a4 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/8b1cedce661a449fa908a6a685fd67a4 2024-11-19T17:14:45,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/dfe2722f821d4d63860f170c16b160ff to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/dfe2722f821d4d63860f170c16b160ff 2024-11-19T17:14:45,462 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/87015f6e98254dbc9cfbb7f8429112d1 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/87015f6e98254dbc9cfbb7f8429112d1 2024-11-19T17:14:45,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/1cc48f6507774e9a91c6e4e4fd143596 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/1cc48f6507774e9a91c6e4e4fd143596 2024-11-19T17:14:45,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/735ffea78d964408b3b7dfbc6a0c41f5 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/735ffea78d964408b3b7dfbc6a0c41f5 2024-11-19T17:14:45,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/fbad9f5a394d4f9186e3679e00c0c2a0 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/fbad9f5a394d4f9186e3679e00c0c2a0 2024-11-19T17:14:45,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/3a8f8850f2a24bc7bff3c8432ea700d2 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/3a8f8850f2a24bc7bff3c8432ea700d2 2024-11-19T17:14:45,470 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/recovered.edits/585.seqid, newMaxSeqId=585, maxSeqId=4 2024-11-19T17:14:45,471 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. 2024-11-19T17:14:45,471 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegion(1676): Region close journal for 5d916b1a7715625884544afede338b7a: Waiting for close lock at 1732036484085Running coprocessor pre-close hooks at 1732036484085Disabling compacts and flushes for region at 1732036484085Disabling writes for close at 1732036484085Obtaining lock to block concurrent updates at 1732036484085Preparing flush snapshotting stores in 5d916b1a7715625884544afede338b7a at 1732036484085Finished memstore snapshotting TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732036484086 (+1 ms)Flushing stores of TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a. at 1732036484086Flushing 5d916b1a7715625884544afede338b7a/A: creating writer at 1732036484086Flushing mob file 5d916b1a7715625884544afede338b7a/A: appending metadata at 1732036484093 (+7 ms)Flushing mob file 5d916b1a7715625884544afede338b7a/A: closing flushed file at 1732036484093Flushing 5d916b1a7715625884544afede338b7a/A: appending metadata at 1732036484504 (+411 ms)Flushing 5d916b1a7715625884544afede338b7a/A: closing flushed file at 1732036484504Flushing 5d916b1a7715625884544afede338b7a/B: creating writer at 1732036484912 (+408 ms)Flushing 5d916b1a7715625884544afede338b7a/B: appending metadata at 1732036484914 (+2 ms)Flushing 5d916b1a7715625884544afede338b7a/B: closing flushed file at 1732036484914Flushing 5d916b1a7715625884544afede338b7a/C: creating writer at 1732036485323 (+409 ms)Flushing 5d916b1a7715625884544afede338b7a/C: appending metadata at 1732036485326 (+3 ms)Flushing 5d916b1a7715625884544afede338b7a/C: closing flushed file at 1732036485326Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d011c49: reopening flushed file at 1732036485339 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@415106f3: reopening flushed file at 1732036485343 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@157a2845: reopening flushed file at 1732036485347 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 5d916b1a7715625884544afede338b7a in 1267ms, sequenceid=582, compaction requested=false at 1732036485352 (+5 ms)Writing region close event to WAL at 1732036485467 (+115 ms)Running coprocessor post-close hooks at 1732036485471 (+4 ms)Closed at 1732036485471 2024-11-19T17:14:45,473 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] handler.UnassignRegionHandler(157): Closed 5d916b1a7715625884544afede338b7a 2024-11-19T17:14:45,473 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=5d916b1a7715625884544afede338b7a, regionState=CLOSED 2024-11-19T17:14:45,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=70, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 because future has completed 2024-11-19T17:14:45,477 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-19T17:14:45,477 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; CloseRegionProcedure 5d916b1a7715625884544afede338b7a, server=280a42d09ec8,33873,1732036422473 in 1.5430 sec 2024-11-19T17:14:45,478 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=70, resume processing ppid=69 2024-11-19T17:14:45,478 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, ppid=69, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d916b1a7715625884544afede338b7a, UNASSIGN in 1.5490 sec 2024-11-19T17:14:45,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-19T17:14:45,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5530 sec 2024-11-19T17:14:45,482 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732036485482"}]},"ts":"1732036485482"} 2024-11-19T17:14:45,483 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T17:14:45,483 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T17:14:45,485 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5660 sec 2024-11-19T17:14:46,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T17:14:46,074 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:14:46,074 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T17:14:46,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:14:46,076 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=72, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:14:46,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T17:14:46,077 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=72, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:14:46,080 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a 2024-11-19T17:14:46,082 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A, FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B, FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C, FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/recovered.edits] 2024-11-19T17:14:46,085 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/81ceb0cc2b9e493abf821017690a071d to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/81ceb0cc2b9e493abf821017690a071d 2024-11-19T17:14:46,086 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/c3c87d3d70db4adab663ec7bfaee8b2c to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/A/c3c87d3d70db4adab663ec7bfaee8b2c 2024-11-19T17:14:46,088 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/d98ef386d5b740e4989635bf913015a8 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/d98ef386d5b740e4989635bf913015a8 2024-11-19T17:14:46,089 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/ec8d7779d626473f91dea7795948450d to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/B/ec8d7779d626473f91dea7795948450d 2024-11-19T17:14:46,092 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/5189713c51024576b96cb42f5fa47823 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/5189713c51024576b96cb42f5fa47823 2024-11-19T17:14:46,093 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/9ef4a6327e2a4b26aa6f01e8bc043d69 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/C/9ef4a6327e2a4b26aa6f01e8bc043d69 2024-11-19T17:14:46,096 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/recovered.edits/585.seqid to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a/recovered.edits/585.seqid 2024-11-19T17:14:46,097 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/5d916b1a7715625884544afede338b7a 2024-11-19T17:14:46,097 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T17:14:46,097 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T17:14:46,098 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T17:14:46,103 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119006b023525aa4052b4472a3abb55aa93_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119006b023525aa4052b4472a3abb55aa93_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:46,104 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119051b1d1e99bf4d708b50cfb0f3a99e5e_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119051b1d1e99bf4d708b50cfb0f3a99e5e_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:46,105 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190f54c21f1ecf4e8fb2716ae405e64567_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190f54c21f1ecf4e8fb2716ae405e64567_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:46,106 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191da4d07a19224fff98096bd8161bac5c_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191da4d07a19224fff98096bd8161bac5c_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:46,107 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192ae2c42a499b48a0b49966d407f0e55f_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192ae2c42a499b48a0b49966d407f0e55f_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:46,108 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192ed6b37ec4714c61b7568dc249644477_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192ed6b37ec4714c61b7568dc249644477_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:46,109 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192f2eb0924ca5456c8432dbea8594033e_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192f2eb0924ca5456c8432dbea8594033e_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:46,110 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194026395cd7534eeeb73c4a5e11927e99_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194026395cd7534eeeb73c4a5e11927e99_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:46,111 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194778a3837ce842d6b519fdaf7369abd6_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194778a3837ce842d6b519fdaf7369abd6_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:46,112 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196ad0bc1681f442b090dc01c33a4d1437_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196ad0bc1681f442b090dc01c33a4d1437_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:46,113 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196ef65326e31443b7b8cd006f40e97489_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196ef65326e31443b7b8cd006f40e97489_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:46,114 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119824688cd745b4014904c5652a1564a66_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119824688cd745b4014904c5652a1564a66_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:46,115 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119824b170d7d9b46ae9e4a0cf6d6782d75_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119824b170d7d9b46ae9e4a0cf6d6782d75_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:46,116 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198aa3245e4c7d409aaefcef848ef8dbda_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198aa3245e4c7d409aaefcef848ef8dbda_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:46,118 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198d0527a55cda4629b9321109b51b7549_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198d0527a55cda4629b9321109b51b7549_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:46,119 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111992a1d804dc8d4c1e849d722574bb0e4f_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111992a1d804dc8d4c1e849d722574bb0e4f_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:46,120 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111992ef67896e644a72b993c5845d8bac71_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111992ef67896e644a72b993c5845d8bac71_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:46,122 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111993b36dc95abe4e7c9a19cf5b8a582aa2_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111993b36dc95abe4e7c9a19cf5b8a582aa2_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:46,123 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119972bec70d67c4d59b2d6454f5f702266_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119972bec70d67c4d59b2d6454f5f702266_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:46,124 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b98eaf2679904e98982d1460426cda10_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b98eaf2679904e98982d1460426cda10_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:46,125 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c0fb4d55af6749e09273892c8e274363_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c0fb4d55af6749e09273892c8e274363_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:46,126 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c22db7fd04704d529eefd3d0e3558c36_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c22db7fd04704d529eefd3d0e3558c36_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:46,127 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ecf3590fd7b34489bee5a001a4b51103_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ecf3590fd7b34489bee5a001a4b51103_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:46,132 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119efe03ddddc744d7d867d7f1daeedd3ab_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119efe03ddddc744d7d867d7f1daeedd3ab_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:46,133 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f56151e64e894e898b2e54931597c673_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f56151e64e894e898b2e54931597c673_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:46,135 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119fe0401679a7b484e9ee33888c6d66a89_5d916b1a7715625884544afede338b7a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119fe0401679a7b484e9ee33888c6d66a89_5d916b1a7715625884544afede338b7a 2024-11-19T17:14:46,135 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T17:14:46,138 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=72, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:14:46,141 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T17:14:46,143 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T17:14:46,144 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=72, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:14:46,144 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T17:14:46,144 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732036486144"}]},"ts":"9223372036854775807"} 2024-11-19T17:14:46,147 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T17:14:46,147 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5d916b1a7715625884544afede338b7a, NAME => 'TestAcidGuarantees,,1732036453156.5d916b1a7715625884544afede338b7a.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T17:14:46,147 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T17:14:46,147 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732036486147"}]},"ts":"9223372036854775807"} 2024-11-19T17:14:46,150 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T17:14:46,151 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=72, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:14:46,152 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 76 msec 2024-11-19T17:14:46,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T17:14:46,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T17:14:46,187 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:14:46,195 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 242) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/cluster_8102c339-c42d-0a52-7e0c-92f0478cebda/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/cluster_8102c339-c42d-0a52-7e0c-92f0478cebda/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_569709797_22 at /127.0.0.1:47410 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=419 (was 344) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6946 (was 7378) 2024-11-19T17:14:46,205 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=419, ProcessCount=11, AvailableMemoryMB=6946 2024-11-19T17:14:46,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T17:14:46,206 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T17:14:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=73, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T17:14:46,209 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=73, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T17:14:46,209 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:46,209 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 73 2024-11-19T17:14:46,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73 2024-11-19T17:14:46,210 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=73, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T17:14:46,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742128_1304 (size=1061) 2024-11-19T17:14:46,217 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e 2024-11-19T17:14:46,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742129_1305 (size=53) 2024-11-19T17:14:46,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73 2024-11-19T17:14:46,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73 2024-11-19T17:14:46,622 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:14:46,622 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c1ce7345860509b228081c672a073b0d, disabling compactions & flushes 2024-11-19T17:14:46,622 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:46,622 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:46,622 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. after waiting 0 ms 2024-11-19T17:14:46,622 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:46,622 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:46,622 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c1ce7345860509b228081c672a073b0d: Waiting for close lock at 1732036486622Disabling compacts and flushes for region at 1732036486622Disabling writes for close at 1732036486622Writing region close event to WAL at 1732036486622Closed at 1732036486622 2024-11-19T17:14:46,624 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=73, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T17:14:46,624 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732036486624"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732036486624"}]},"ts":"1732036486624"} 2024-11-19T17:14:46,625 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T17:14:46,626 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=73, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T17:14:46,626 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732036486626"}]},"ts":"1732036486626"} 2024-11-19T17:14:46,627 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T17:14:46,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1ce7345860509b228081c672a073b0d, ASSIGN}] 2024-11-19T17:14:46,629 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1ce7345860509b228081c672a073b0d, ASSIGN 2024-11-19T17:14:46,629 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1ce7345860509b228081c672a073b0d, ASSIGN; state=OFFLINE, location=280a42d09ec8,33873,1732036422473; forceNewPlan=false, retain=false 2024-11-19T17:14:46,780 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=c1ce7345860509b228081c672a073b0d, regionState=OPENING, regionLocation=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:46,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1ce7345860509b228081c672a073b0d, ASSIGN because future has completed 2024-11-19T17:14:46,783 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473}] 2024-11-19T17:14:46,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73 2024-11-19T17:14:46,938 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:46,938 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:14:46,939 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c1ce7345860509b228081c672a073b0d 2024-11-19T17:14:46,939 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:14:46,939 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for c1ce7345860509b228081c672a073b0d 2024-11-19T17:14:46,939 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for c1ce7345860509b228081c672a073b0d 2024-11-19T17:14:46,940 INFO [StoreOpener-c1ce7345860509b228081c672a073b0d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c1ce7345860509b228081c672a073b0d 2024-11-19T17:14:46,941 INFO [StoreOpener-c1ce7345860509b228081c672a073b0d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T17:14:46,942 INFO [StoreOpener-c1ce7345860509b228081c672a073b0d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1ce7345860509b228081c672a073b0d columnFamilyName A 2024-11-19T17:14:46,942 DEBUG [StoreOpener-c1ce7345860509b228081c672a073b0d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:46,942 INFO [StoreOpener-c1ce7345860509b228081c672a073b0d-1 {}] regionserver.HStore(327): Store=c1ce7345860509b228081c672a073b0d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:14:46,942 INFO [StoreOpener-c1ce7345860509b228081c672a073b0d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c1ce7345860509b228081c672a073b0d 2024-11-19T17:14:46,943 INFO [StoreOpener-c1ce7345860509b228081c672a073b0d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T17:14:46,943 INFO [StoreOpener-c1ce7345860509b228081c672a073b0d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1ce7345860509b228081c672a073b0d columnFamilyName B 2024-11-19T17:14:46,943 DEBUG [StoreOpener-c1ce7345860509b228081c672a073b0d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:46,944 INFO [StoreOpener-c1ce7345860509b228081c672a073b0d-1 {}] regionserver.HStore(327): Store=c1ce7345860509b228081c672a073b0d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:14:46,944 INFO [StoreOpener-c1ce7345860509b228081c672a073b0d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c1ce7345860509b228081c672a073b0d 2024-11-19T17:14:46,945 INFO [StoreOpener-c1ce7345860509b228081c672a073b0d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T17:14:46,946 INFO [StoreOpener-c1ce7345860509b228081c672a073b0d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1ce7345860509b228081c672a073b0d columnFamilyName C 2024-11-19T17:14:46,946 DEBUG [StoreOpener-c1ce7345860509b228081c672a073b0d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:14:46,946 INFO [StoreOpener-c1ce7345860509b228081c672a073b0d-1 {}] regionserver.HStore(327): Store=c1ce7345860509b228081c672a073b0d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:14:46,946 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:46,946 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for c1ce7345860509b228081c672a073b0d 2024-11-19T17:14:46,947 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d 2024-11-19T17:14:46,947 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d 2024-11-19T17:14:46,948 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for c1ce7345860509b228081c672a073b0d 2024-11-19T17:14:46,948 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for c1ce7345860509b228081c672a073b0d 2024-11-19T17:14:46,949 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T17:14:46,951 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for c1ce7345860509b228081c672a073b0d 2024-11-19T17:14:46,954 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T17:14:46,959 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened c1ce7345860509b228081c672a073b0d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67810737, jitterRate=0.010458722710609436}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T17:14:46,959 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c1ce7345860509b228081c672a073b0d 2024-11-19T17:14:46,959 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for c1ce7345860509b228081c672a073b0d: Running coprocessor pre-open hook at 1732036486939Writing region info on filesystem at 1732036486939Initializing all the Stores at 1732036486940 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036486940Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036486940Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036486940Cleaning up temporary data from old regions at 1732036486948 (+8 ms)Running coprocessor post-open hooks at 1732036486959 (+11 ms)Region opened successfully at 1732036486959 2024-11-19T17:14:46,960 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., pid=75, masterSystemTime=1732036486935 2024-11-19T17:14:46,962 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:46,962 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:46,963 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=c1ce7345860509b228081c672a073b0d, regionState=OPEN, openSeqNum=2, regionLocation=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:46,966 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 because future has completed 2024-11-19T17:14:46,966 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40603 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=280a42d09ec8,33873,1732036422473, table=TestAcidGuarantees, region=c1ce7345860509b228081c672a073b0d. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T17:14:46,969 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-19T17:14:46,969 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; OpenRegionProcedure c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 in 183 msec 2024-11-19T17:14:46,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=74, resume processing ppid=73 2024-11-19T17:14:46,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, ppid=73, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1ce7345860509b228081c672a073b0d, ASSIGN in 341 msec 2024-11-19T17:14:46,972 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=73, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T17:14:46,972 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732036486972"}]},"ts":"1732036486972"} 2024-11-19T17:14:46,973 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T17:14:46,974 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=73, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T17:14:46,976 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 768 msec 2024-11-19T17:14:47,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73 2024-11-19T17:14:47,344 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:14:47,345 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe11fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,345 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:14:47,345 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:14:47,346 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:14:47,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:14:47,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:14:47,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@777fb864, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:14:47,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:14:47,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:47,348 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:14:47,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13e60fc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,349 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:14:47,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:14:47,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:14:47,352 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:14:47,353 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T17:14:47,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 280a42d09ec8,40603,1732036421715 2024-11-19T17:14:47,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@448f22f3 2024-11-19T17:14:47,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T17:14:47,354 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T17:14:47,355 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,355 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:14:47,356 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:14:47,356 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:14:47,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:14:47,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:14:47,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40477d54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:14:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:14:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:47,357 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:14:47,358 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@521f3f7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,358 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,358 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:14:47,358 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:14:47,359 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:14:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:14:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:14:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e08b9ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:14:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:14:47,360 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:47,360 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:14:47,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3534b4ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,361 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,361 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:14:47,361 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:14:47,362 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:14:47,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:14:47,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:14:47,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@515b7bcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:14:47,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:14:47,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:47,363 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:14:47,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@792792c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,364 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,364 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:14:47,364 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:14:47,364 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:14:47,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:14:47,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:14:47,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@571e7405, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:14:47,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:14:47,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:47,365 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:14:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b515727, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,366 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,366 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:14:47,366 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:14:47,367 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:14:47,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:14:47,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:14:47,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c3db875, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:14:47,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:14:47,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:47,368 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:14:47,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@691f30e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,369 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,369 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:14:47,369 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:14:47,369 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:14:47,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:14:47,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:14:47,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57453ffe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:14:47,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:14:47,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:47,371 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51710, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:14:47,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@356c305b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,371 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,372 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:14:47,372 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:14:47,372 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:14:47,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:14:47,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:14:47,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a841559, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:14:47,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:14:47,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:47,373 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:14:47,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56cf5eca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,374 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,374 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:14:47,374 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:14:47,375 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:14:47,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:14:47,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:14:47,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37dff720, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:14:47,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:14:47,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:47,376 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:14:47,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b0e2e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,377 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,377 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:14:47,377 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:14:47,378 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:14:47,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:14:47,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:14:47,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@461c51bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:14:47,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:14:47,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:47,379 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:14:47,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34c6c6c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,380 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728d338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,380 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:14:47,380 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:14:47,380 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:14:47,380 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:14:47,380 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:14:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cd65f1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:14:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:14:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:14:47,381 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:14:47,382 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cf2ef48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:14:47,382 DEBUG [Thread-1383 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:14:47,382 DEBUG [Thread-1390 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:14:47,383 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:14:47,383 DEBUG [Thread-1392 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:14:47,383 DEBUG [Thread-1396 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:14:47,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:14:47,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:14:47,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:14:47,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:14:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:14:47,384 DEBUG [Thread-1398 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:14:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:14:47,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-19T17:14:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:14:47,384 DEBUG [Thread-1379 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:14:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:14:47,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:14:47,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:14:47,385 DEBUG [Thread-1381 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:14:47,385 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:14:47,385 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:14:47,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T17:14:47,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:14:47,386 DEBUG [Thread-1387 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:14:47,386 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:14:47,386 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:14:47,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:14:47,386 DEBUG [Thread-1385 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:14:47,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:14:47,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:14:47,386 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60474, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:14:47,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:14:47,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:14:47,387 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:14:47,387 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:14:47,387 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:14:47,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:14:47,388 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:14:47,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2] 2024-11-19T17:14:47,388 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:14:47,388 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:14:47,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:14:47,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2] 2024-11-19T17:14:47,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2] 2024-11-19T17:14:47,389 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:14:47,390 DEBUG [Thread-1394 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:14:47,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2] 2024-11-19T17:14:47,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2] 2024-11-19T17:14:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:14:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2] 2024-11-19T17:14:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:14:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2] 2024-11-19T17:14:47,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2] 2024-11-19T17:14:47,393 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:14:47,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2] 2024-11-19T17:14:47,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2] 2024-11-19T17:14:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on c1ce7345860509b228081c672a073b0d 2024-11-19T17:14:47,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1ce7345860509b228081c672a073b0d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T17:14:47,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=A 2024-11-19T17:14:47,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:47,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=B 2024-11-19T17:14:47,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:47,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=C 2024-11-19T17:14:47,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:47,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:47,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036547433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:47,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:47,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:47,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:47,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:47,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036547439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:47,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:47,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:47,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:47,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036547441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:47,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:47,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:47,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036547441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:47,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:47,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:47,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036547441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:47,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/f6bead26a6eb41f393736023ae51c14b is 50, key is test_row_0/A:col10/1732036487402/Put/seqid=0 2024-11-19T17:14:47,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742130_1306 (size=12001) 2024-11-19T17:14:47,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/f6bead26a6eb41f393736023ae51c14b 2024-11-19T17:14:47,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T17:14:47,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/ed15f3d24a5c4472914fd4a3ddfa65b1 is 50, key is test_row_0/B:col10/1732036487402/Put/seqid=0 2024-11-19T17:14:47,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742131_1307 (size=12001) 2024-11-19T17:14:47,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T17:14:47,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:47,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:14:47,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:47,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:47,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:47,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:47,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036547554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:47,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036547554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:47,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036547554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:47,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036547554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:47,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036547555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:47,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T17:14:47,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:47,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:14:47,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:47,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:47,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T17:14:47,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036547764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:47,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:47,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:47,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036547764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:47,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:47,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:47,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:47,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:47,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:47,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036547764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:47,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:47,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:47,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036547764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:47,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:47,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:47,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:47,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036547765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:47,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:47,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:47,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:47,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:47,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:47,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:47,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T17:14:47,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:47,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:14:47,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:47,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:47,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:47,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:47,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/ed15f3d24a5c4472914fd4a3ddfa65b1 2024-11-19T17:14:47,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/faf4c31fffe04a1897fc1f390b1314ee is 50, key is test_row_0/C:col10/1732036487402/Put/seqid=0 2024-11-19T17:14:47,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742132_1308 (size=12001) 2024-11-19T17:14:48,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T17:14:48,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:48,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:14:48,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:48,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:48,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:48,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:48,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T17:14:48,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:48,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036548074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:48,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:48,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036548074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:48,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:48,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036548074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:48,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:48,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:48,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036548074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:48,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:48,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:48,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036548074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:48,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:48,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T17:14:48,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:48,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:14:48,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:48,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:48,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:48,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:48,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T17:14:48,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:48,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:14:48,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:48,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:48,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:48,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:48,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/faf4c31fffe04a1897fc1f390b1314ee 2024-11-19T17:14:48,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/f6bead26a6eb41f393736023ae51c14b as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/f6bead26a6eb41f393736023ae51c14b 2024-11-19T17:14:48,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/f6bead26a6eb41f393736023ae51c14b, entries=150, sequenceid=13, filesize=11.7 K 2024-11-19T17:14:48,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/ed15f3d24a5c4472914fd4a3ddfa65b1 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/ed15f3d24a5c4472914fd4a3ddfa65b1 2024-11-19T17:14:48,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/ed15f3d24a5c4472914fd4a3ddfa65b1, entries=150, sequenceid=13, filesize=11.7 K 2024-11-19T17:14:48,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/faf4c31fffe04a1897fc1f390b1314ee as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/faf4c31fffe04a1897fc1f390b1314ee 2024-11-19T17:14:48,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/faf4c31fffe04a1897fc1f390b1314ee, entries=150, sequenceid=13, filesize=11.7 K 2024-11-19T17:14:48,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c1ce7345860509b228081c672a073b0d in 972ms, sequenceid=13, compaction requested=false 2024-11-19T17:14:48,380 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T17:14:48,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:14:48,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T17:14:48,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:48,463 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing c1ce7345860509b228081c672a073b0d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T17:14:48,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=A 2024-11-19T17:14:48,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:48,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=B 2024-11-19T17:14:48,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:48,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=C 2024-11-19T17:14:48,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:48,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/7597280727a84e2ea8cee5784dbeaae5 is 50, key is test_row_0/A:col10/1732036487430/Put/seqid=0 2024-11-19T17:14:48,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742133_1309 (size=12001) 2024-11-19T17:14:48,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T17:14:48,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:14:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on c1ce7345860509b228081c672a073b0d 2024-11-19T17:14:48,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:48,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036548593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:48,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036548594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:48,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036548595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:48,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:48,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:48,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036548597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:48,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:48,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:48,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036548597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:48,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:48,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:48,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036548704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:48,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:48,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036548704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:48,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036548704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:48,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036548704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:48,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036548704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:48,872 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/7597280727a84e2ea8cee5784dbeaae5 2024-11-19T17:14:48,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/c57894828ce74820a443cb923d02e053 is 50, key is test_row_0/B:col10/1732036487430/Put/seqid=0 2024-11-19T17:14:48,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742134_1310 (size=12001) 2024-11-19T17:14:48,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:48,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036548914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:48,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:48,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036548914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:48,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:48,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036548914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:48,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036548914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:48,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036548914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:49,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:49,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:49,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036549223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:49,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036549224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:49,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:49,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:49,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:49,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:49,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:49,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036549224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:49,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036549224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:49,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:49,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036549224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:49,286 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/c57894828ce74820a443cb923d02e053 2024-11-19T17:14:49,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/2c966e10909447ecbc8c241ad88517df is 50, key is test_row_0/C:col10/1732036487430/Put/seqid=0 2024-11-19T17:14:49,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742135_1311 (size=12001) 2024-11-19T17:14:49,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T17:14:49,701 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/2c966e10909447ecbc8c241ad88517df 2024-11-19T17:14:49,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/7597280727a84e2ea8cee5784dbeaae5 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/7597280727a84e2ea8cee5784dbeaae5 2024-11-19T17:14:49,710 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/7597280727a84e2ea8cee5784dbeaae5, entries=150, sequenceid=33, filesize=11.7 K 2024-11-19T17:14:49,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/c57894828ce74820a443cb923d02e053 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/c57894828ce74820a443cb923d02e053 2024-11-19T17:14:49,717 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/c57894828ce74820a443cb923d02e053, entries=150, sequenceid=33, filesize=11.7 K 2024-11-19T17:14:49,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/2c966e10909447ecbc8c241ad88517df as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/2c966e10909447ecbc8c241ad88517df 2024-11-19T17:14:49,722 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/2c966e10909447ecbc8c241ad88517df, entries=150, sequenceid=33, filesize=11.7 K 2024-11-19T17:14:49,723 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for c1ce7345860509b228081c672a073b0d in 1261ms, sequenceid=33, compaction requested=false 2024-11-19T17:14:49,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:14:49,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:49,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-19T17:14:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-19T17:14:49,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-19T17:14:49,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3370 sec 2024-11-19T17:14:49,729 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.3450 sec 2024-11-19T17:14:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on c1ce7345860509b228081c672a073b0d 2024-11-19T17:14:49,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1ce7345860509b228081c672a073b0d 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T17:14:49,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=A 2024-11-19T17:14:49,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:49,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=B 2024-11-19T17:14:49,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:49,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=C 2024-11-19T17:14:49,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:49,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/fdc1cddfed71443dad8089e46ea656d7 is 50, key is test_row_0/A:col10/1732036489735/Put/seqid=0 2024-11-19T17:14:49,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742136_1312 (size=12001) 2024-11-19T17:14:49,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/fdc1cddfed71443dad8089e46ea656d7 2024-11-19T17:14:49,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:49,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036549753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:49,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:49,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:49,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:49,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036549754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:49,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:49,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:49,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:49,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:49,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/9eef32356aa1469bb0e5bf7ed405f59e is 50, key is test_row_0/B:col10/1732036489735/Put/seqid=0 2024-11-19T17:14:49,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:49,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036549760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:49,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:49,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:49,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:49,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:49,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036549762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:49,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:49,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:49,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:49,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:49,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036549762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:49,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:49,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:49,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:49,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742137_1313 (size=12001) 2024-11-19T17:14:49,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/9eef32356aa1469bb0e5bf7ed405f59e 2024-11-19T17:14:49,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/77f7463d58f34856a49da6efb672db16 is 50, key is test_row_0/C:col10/1732036489735/Put/seqid=0 2024-11-19T17:14:49,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742138_1314 (size=12001) 2024-11-19T17:14:49,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:49,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036549874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:49,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:49,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036549874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:49,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:49,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:49,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036549874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:49,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:49,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:49,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:49,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:49,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036549874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:49,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:49,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:49,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:49,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:49,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:49,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036549874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:49,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:49,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:49,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:49,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:50,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:50,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036550084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:50,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:50,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036550084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:50,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:50,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:50,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036550084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:50,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036550084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:50,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036550084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:50,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:50,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/77f7463d58f34856a49da6efb672db16 2024-11-19T17:14:50,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/fdc1cddfed71443dad8089e46ea656d7 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/fdc1cddfed71443dad8089e46ea656d7 2024-11-19T17:14:50,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/fdc1cddfed71443dad8089e46ea656d7, entries=150, sequenceid=48, filesize=11.7 K 2024-11-19T17:14:50,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/9eef32356aa1469bb0e5bf7ed405f59e as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/9eef32356aa1469bb0e5bf7ed405f59e 2024-11-19T17:14:50,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/9eef32356aa1469bb0e5bf7ed405f59e, entries=150, sequenceid=48, filesize=11.7 K 2024-11-19T17:14:50,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/77f7463d58f34856a49da6efb672db16 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/77f7463d58f34856a49da6efb672db16 2024-11-19T17:14:50,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/77f7463d58f34856a49da6efb672db16, entries=150, sequenceid=48, filesize=11.7 K 2024-11-19T17:14:50,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c1ce7345860509b228081c672a073b0d in 476ms, sequenceid=48, compaction requested=true 2024-11-19T17:14:50,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:14:50,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1ce7345860509b228081c672a073b0d:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:14:50,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:50,212 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:50,212 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:50,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1ce7345860509b228081c672a073b0d:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:14:50,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:50,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1ce7345860509b228081c672a073b0d:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:14:50,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:50,216 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:50,216 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): c1ce7345860509b228081c672a073b0d/A is initiating minor compaction (all files) 2024-11-19T17:14:50,216 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:50,216 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): c1ce7345860509b228081c672a073b0d/B is initiating minor compaction (all files) 2024-11-19T17:14:50,216 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1ce7345860509b228081c672a073b0d/A in TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:50,216 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1ce7345860509b228081c672a073b0d/B in TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:50,216 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/f6bead26a6eb41f393736023ae51c14b, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/7597280727a84e2ea8cee5784dbeaae5, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/fdc1cddfed71443dad8089e46ea656d7] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp, totalSize=35.2 K 2024-11-19T17:14:50,216 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/ed15f3d24a5c4472914fd4a3ddfa65b1, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/c57894828ce74820a443cb923d02e053, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/9eef32356aa1469bb0e5bf7ed405f59e] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp, totalSize=35.2 K 2024-11-19T17:14:50,217 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6bead26a6eb41f393736023ae51c14b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732036487392 2024-11-19T17:14:50,217 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting ed15f3d24a5c4472914fd4a3ddfa65b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732036487392 2024-11-19T17:14:50,218 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting c57894828ce74820a443cb923d02e053, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732036487430 2024-11-19T17:14:50,218 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7597280727a84e2ea8cee5784dbeaae5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732036487430 2024-11-19T17:14:50,218 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdc1cddfed71443dad8089e46ea656d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732036488591 2024-11-19T17:14:50,218 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 9eef32356aa1469bb0e5bf7ed405f59e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732036488591 2024-11-19T17:14:50,231 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1ce7345860509b228081c672a073b0d#B#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:50,231 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1ce7345860509b228081c672a073b0d#A#compaction#263 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:50,231 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/145ead5330ee4f23a4426aa03571580a is 50, key is test_row_0/B:col10/1732036489735/Put/seqid=0 2024-11-19T17:14:50,231 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/4b665a00db49462ba39db5d309285067 is 50, key is test_row_0/A:col10/1732036489735/Put/seqid=0 2024-11-19T17:14:50,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742139_1315 (size=12104) 2024-11-19T17:14:50,261 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/145ead5330ee4f23a4426aa03571580a as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/145ead5330ee4f23a4426aa03571580a 2024-11-19T17:14:50,268 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1ce7345860509b228081c672a073b0d/B of c1ce7345860509b228081c672a073b0d into 145ead5330ee4f23a4426aa03571580a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:50,268 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:14:50,268 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., storeName=c1ce7345860509b228081c672a073b0d/B, priority=13, startTime=1732036490212; duration=0sec 2024-11-19T17:14:50,268 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:50,268 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1ce7345860509b228081c672a073b0d:B 2024-11-19T17:14:50,268 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:50,269 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:50,269 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): c1ce7345860509b228081c672a073b0d/C is initiating minor compaction (all files) 2024-11-19T17:14:50,270 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1ce7345860509b228081c672a073b0d/C in TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:50,270 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/faf4c31fffe04a1897fc1f390b1314ee, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/2c966e10909447ecbc8c241ad88517df, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/77f7463d58f34856a49da6efb672db16] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp, totalSize=35.2 K 2024-11-19T17:14:50,270 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting faf4c31fffe04a1897fc1f390b1314ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732036487392 2024-11-19T17:14:50,271 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c966e10909447ecbc8c241ad88517df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732036487430 2024-11-19T17:14:50,271 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 77f7463d58f34856a49da6efb672db16, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732036488591 2024-11-19T17:14:50,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742140_1316 (size=12104) 2024-11-19T17:14:50,281 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1ce7345860509b228081c672a073b0d#C#compaction#264 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:50,281 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/49d9d1f76f664682a238f4cc502ca439 is 50, key is test_row_0/C:col10/1732036489735/Put/seqid=0 2024-11-19T17:14:50,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742141_1317 (size=12104) 2024-11-19T17:14:50,294 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/49d9d1f76f664682a238f4cc502ca439 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/49d9d1f76f664682a238f4cc502ca439 2024-11-19T17:14:50,314 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1ce7345860509b228081c672a073b0d/C of c1ce7345860509b228081c672a073b0d into 49d9d1f76f664682a238f4cc502ca439(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:50,314 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:14:50,315 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., storeName=c1ce7345860509b228081c672a073b0d/C, priority=13, startTime=1732036490213; duration=0sec 2024-11-19T17:14:50,315 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:50,315 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1ce7345860509b228081c672a073b0d:C 2024-11-19T17:14:50,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on c1ce7345860509b228081c672a073b0d 2024-11-19T17:14:50,397 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1ce7345860509b228081c672a073b0d 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T17:14:50,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=A 2024-11-19T17:14:50,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:50,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=B 2024-11-19T17:14:50,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:50,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=C 2024-11-19T17:14:50,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:50,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/b2754535260f4b83a38436038340b076 is 50, key is test_row_0/A:col10/1732036490396/Put/seqid=0 2024-11-19T17:14:50,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742142_1318 (size=12001) 2024-11-19T17:14:50,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:50,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036550408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:50,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:50,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:50,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036550414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:50,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:50,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036550416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:50,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:50,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:50,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036550416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:50,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:50,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036550416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:50,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:50,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:50,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:50,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:50,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036550524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:50,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:50,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:50,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036550524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:50,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:50,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036550524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:50,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:50,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:50,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:50,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036550533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:50,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:50,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036550533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:50,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:50,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:50,694 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/4b665a00db49462ba39db5d309285067 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/4b665a00db49462ba39db5d309285067 2024-11-19T17:14:50,699 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1ce7345860509b228081c672a073b0d/A of c1ce7345860509b228081c672a073b0d into 4b665a00db49462ba39db5d309285067(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:50,699 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:14:50,699 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., storeName=c1ce7345860509b228081c672a073b0d/A, priority=13, startTime=1732036490212; duration=0sec 2024-11-19T17:14:50,699 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:50,699 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1ce7345860509b228081c672a073b0d:A 2024-11-19T17:14:50,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036550734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:50,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036550734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:50,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036550734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:50,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:50,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036550743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:50,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:50,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036550743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:50,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:50,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:50,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/b2754535260f4b83a38436038340b076 2024-11-19T17:14:50,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/f50d59af2fa74de4ae5ae65d59bec3ce is 50, key is test_row_0/B:col10/1732036490396/Put/seqid=0 2024-11-19T17:14:50,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742143_1319 (size=12001) 2024-11-19T17:14:51,001 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T17:14:51,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:51,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036551043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:51,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:51,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:51,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:51,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:51,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036551044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:51,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:51,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:51,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:51,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036551044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:51,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:51,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:51,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:51,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:51,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036551054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:51,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:51,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:51,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:51,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036551054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:51,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:51,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:51,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:51,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:51,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/f50d59af2fa74de4ae5ae65d59bec3ce 2024-11-19T17:14:51,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/9c398c079d0247a3842a6462d14d8b4c is 50, key is test_row_0/C:col10/1732036490396/Put/seqid=0 2024-11-19T17:14:51,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742144_1320 (size=12001) 2024-11-19T17:14:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T17:14:51,554 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:14:51,554 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:14:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-19T17:14:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T17:14:51,556 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:14:51,557 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:14:51,557 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:14:51,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:51,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036551554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036551554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:51,558 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:51,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:51,558 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:51,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:51,558 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:51,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:51,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036551563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:51,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:51,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:51,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:51,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036551563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:51,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:51,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:51,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:51,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036551573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:51,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/9c398c079d0247a3842a6462d14d8b4c 2024-11-19T17:14:51,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/b2754535260f4b83a38436038340b076 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/b2754535260f4b83a38436038340b076 2024-11-19T17:14:51,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/b2754535260f4b83a38436038340b076, entries=150, sequenceid=70, filesize=11.7 K 2024-11-19T17:14:51,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/f50d59af2fa74de4ae5ae65d59bec3ce as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/f50d59af2fa74de4ae5ae65d59bec3ce 2024-11-19T17:14:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T17:14:51,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/f50d59af2fa74de4ae5ae65d59bec3ce, entries=150, sequenceid=70, filesize=11.7 K 2024-11-19T17:14:51,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/9c398c079d0247a3842a6462d14d8b4c as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/9c398c079d0247a3842a6462d14d8b4c 2024-11-19T17:14:51,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/9c398c079d0247a3842a6462d14d8b4c, entries=150, sequenceid=70, filesize=11.7 K 2024-11-19T17:14:51,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=73.80 KB/75570 for c1ce7345860509b228081c672a073b0d in 1274ms, sequenceid=70, compaction requested=false 2024-11-19T17:14:51,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:14:51,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T17:14:51,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:51,710 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing c1ce7345860509b228081c672a073b0d 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T17:14:51,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=A 2024-11-19T17:14:51,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:51,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=B 2024-11-19T17:14:51,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:51,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=C 2024-11-19T17:14:51,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:51,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/119d0817e1ac43c48e8a2f4e37278fa7 is 50, key is test_row_0/A:col10/1732036490413/Put/seqid=0 2024-11-19T17:14:51,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742145_1321 (size=12001) 2024-11-19T17:14:51,721 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/119d0817e1ac43c48e8a2f4e37278fa7 2024-11-19T17:14:51,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/1e9fec8ad3dc42e187fb02a2af8fa14c is 50, key is test_row_0/B:col10/1732036490413/Put/seqid=0 2024-11-19T17:14:51,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742146_1322 (size=12001) 2024-11-19T17:14:51,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T17:14:52,151 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/1e9fec8ad3dc42e187fb02a2af8fa14c 2024-11-19T17:14:52,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/0d1da9f998c74dcbae934a1d5bd5584b is 50, key is test_row_0/C:col10/1732036490413/Put/seqid=0 2024-11-19T17:14:52,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742147_1323 (size=12001) 2024-11-19T17:14:52,179 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/0d1da9f998c74dcbae934a1d5bd5584b 2024-11-19T17:14:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T17:14:52,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/119d0817e1ac43c48e8a2f4e37278fa7 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/119d0817e1ac43c48e8a2f4e37278fa7 2024-11-19T17:14:52,188 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/119d0817e1ac43c48e8a2f4e37278fa7, entries=150, sequenceid=85, filesize=11.7 K 2024-11-19T17:14:52,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/1e9fec8ad3dc42e187fb02a2af8fa14c as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/1e9fec8ad3dc42e187fb02a2af8fa14c 2024-11-19T17:14:52,193 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/1e9fec8ad3dc42e187fb02a2af8fa14c, entries=150, sequenceid=85, filesize=11.7 K 2024-11-19T17:14:52,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/0d1da9f998c74dcbae934a1d5bd5584b as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/0d1da9f998c74dcbae934a1d5bd5584b 2024-11-19T17:14:52,199 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/0d1da9f998c74dcbae934a1d5bd5584b, entries=150, sequenceid=85, filesize=11.7 K 2024-11-19T17:14:52,200 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for c1ce7345860509b228081c672a073b0d in 489ms, sequenceid=85, compaction requested=true 2024-11-19T17:14:52,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:14:52,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:52,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-19T17:14:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-19T17:14:52,203 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-19T17:14:52,203 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 644 msec 2024-11-19T17:14:52,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 650 msec 2024-11-19T17:14:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on c1ce7345860509b228081c672a073b0d 2024-11-19T17:14:52,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1ce7345860509b228081c672a073b0d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T17:14:52,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=A 2024-11-19T17:14:52,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:52,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=B 2024-11-19T17:14:52,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:52,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=C 2024-11-19T17:14:52,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:52,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/6178faa8886a4e85b7f412577cdb426d is 50, key is test_row_1/A:col10/1732036492586/Put/seqid=0 2024-11-19T17:14:52,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742148_1324 (size=9657) 2024-11-19T17:14:52,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036552607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:52,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036492592, max timestamp=1732036492611Num uniques 859; 2024-11-19T17:14:52,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:14:52,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:52,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036492592, max timestamp=1732036492611Num uniques 859; 2024-11-19T17:14:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036552608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:52,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036492592, max timestamp=1732036492611Num uniques 859; 2024-11-19T17:14:52,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:14:52,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:14:52,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:52,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:52,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:52,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:52,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:52,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:52,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:52,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036552669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:52,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:52,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:52,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:52,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036552670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:52,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:52,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:52,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036552672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:52,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:52,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:52,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:52,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:52,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:52,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:52,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T17:14:52,693 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:14:52,694 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:14:52,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-19T17:14:52,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T17:14:52,695 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:14:52,696 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:14:52,696 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:14:52,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:52,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036552724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:52,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:52,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036552724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:52,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:52,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:52,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:52,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:52,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:52,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:52,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:52,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036552783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:52,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:52,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:52,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036552783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:52,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:52,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:52,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:52,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:52,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:52,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:52,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036552784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:52,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:52,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:52,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T17:14:52,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T17:14:52,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:52,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:14:52,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:52,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:52,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:52,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:52,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:52,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036552934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:52,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:52,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:52,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036552934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:52,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:52,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:52,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:52,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:52,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:52,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:52,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036552994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:52,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:52,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:52,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036552994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:52,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:52,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:52,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:52,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:52,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:52,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036552994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:52,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:52,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:52,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:52,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:53,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T17:14:53,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:53,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:14:53,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:53,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:53,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:53,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/6178faa8886a4e85b7f412577cdb426d 2024-11-19T17:14:53,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:53,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/bed3915a70de42f086edb584d7ba4f27 is 50, key is test_row_1/B:col10/1732036492586/Put/seqid=0 2024-11-19T17:14:53,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742149_1325 (size=9657) 2024-11-19T17:14:53,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T17:14:53,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T17:14:53,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:53,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:14:53,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:53,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:53,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:53,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:53,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:53,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036553243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:53,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:53,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:53,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:53,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036553243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:53,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:53,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:53,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:53,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:53,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:53,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036553303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:53,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:53,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:53,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036553303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:53,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:53,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:53,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:53,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:53,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036553303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:53,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:53,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:53,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:53,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:53,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:53,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T17:14:53,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:53,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:14:53,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:53,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:53,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:53,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:53,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T17:14:53,414 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/bed3915a70de42f086edb584d7ba4f27 2024-11-19T17:14:53,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/f47db503b7674b2d8e95aa1ecdcc05d4 is 50, key is test_row_1/C:col10/1732036492586/Put/seqid=0 2024-11-19T17:14:53,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742150_1326 (size=9657) 2024-11-19T17:14:53,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T17:14:53,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:53,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:14:53,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:53,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:53,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:53,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:53,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T17:14:53,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:53,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:14:53,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:53,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:53,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:53,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:53,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:53,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036553754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:53,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:53,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036553754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:53,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:53,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:53,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:53,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:53,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:53,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:53,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T17:14:53,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:53,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:14:53,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:53,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:53,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:53,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:53,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:53,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036553814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:53,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:53,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:53,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036553814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:53,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:53,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:53,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:53,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:53,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:53,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036553814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:53,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:53,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:53,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:53,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:53,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/f47db503b7674b2d8e95aa1ecdcc05d4 2024-11-19T17:14:53,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/6178faa8886a4e85b7f412577cdb426d as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/6178faa8886a4e85b7f412577cdb426d 2024-11-19T17:14:53,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T17:14:53,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/6178faa8886a4e85b7f412577cdb426d, entries=100, sequenceid=96, filesize=9.4 K 2024-11-19T17:14:53,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/bed3915a70de42f086edb584d7ba4f27 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/bed3915a70de42f086edb584d7ba4f27 2024-11-19T17:14:53,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/bed3915a70de42f086edb584d7ba4f27, entries=100, sequenceid=96, filesize=9.4 K 2024-11-19T17:14:53,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/f47db503b7674b2d8e95aa1ecdcc05d4 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/f47db503b7674b2d8e95aa1ecdcc05d4 2024-11-19T17:14:53,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/f47db503b7674b2d8e95aa1ecdcc05d4, entries=100, sequenceid=96, filesize=9.4 K 2024-11-19T17:14:53,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for c1ce7345860509b228081c672a073b0d in 1261ms, sequenceid=96, compaction requested=true 2024-11-19T17:14:53,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:14:53,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1ce7345860509b228081c672a073b0d:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:14:53,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:53,847 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:14:53,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1ce7345860509b228081c672a073b0d:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:14:53,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:53,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1ce7345860509b228081c672a073b0d:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:14:53,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:53,847 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:14:53,849 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:14:53,849 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:14:53,849 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): c1ce7345860509b228081c672a073b0d/B is initiating minor compaction (all files) 2024-11-19T17:14:53,849 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): c1ce7345860509b228081c672a073b0d/A is initiating minor compaction (all files) 2024-11-19T17:14:53,849 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1ce7345860509b228081c672a073b0d/A in TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:53,849 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1ce7345860509b228081c672a073b0d/B in TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:53,849 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/145ead5330ee4f23a4426aa03571580a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/f50d59af2fa74de4ae5ae65d59bec3ce, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/1e9fec8ad3dc42e187fb02a2af8fa14c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/bed3915a70de42f086edb584d7ba4f27] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp, totalSize=44.7 K 2024-11-19T17:14:53,849 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/4b665a00db49462ba39db5d309285067, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/b2754535260f4b83a38436038340b076, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/119d0817e1ac43c48e8a2f4e37278fa7, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/6178faa8886a4e85b7f412577cdb426d] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp, totalSize=44.7 K 2024-11-19T17:14:53,849 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 145ead5330ee4f23a4426aa03571580a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732036488591 2024-11-19T17:14:53,849 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b665a00db49462ba39db5d309285067, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732036488591 2024-11-19T17:14:53,850 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2754535260f4b83a38436038340b076, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1732036490396 2024-11-19T17:14:53,850 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting f50d59af2fa74de4ae5ae65d59bec3ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1732036490396 2024-11-19T17:14:53,850 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e9fec8ad3dc42e187fb02a2af8fa14c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732036490413 2024-11-19T17:14:53,850 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 119d0817e1ac43c48e8a2f4e37278fa7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732036490413 2024-11-19T17:14:53,850 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6178faa8886a4e85b7f412577cdb426d, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732036492585 2024-11-19T17:14:53,850 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting bed3915a70de42f086edb584d7ba4f27, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732036492585 2024-11-19T17:14:53,860 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1ce7345860509b228081c672a073b0d#A#compaction#274 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:53,860 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1ce7345860509b228081c672a073b0d#B#compaction#275 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:53,860 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/e575b39b5e6f43569445a8fb9bc1a933 is 50, key is test_row_0/B:col10/1732036490413/Put/seqid=0 2024-11-19T17:14:53,860 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/2d98932586504a0c8ebe21b513666bd7 is 50, key is test_row_0/A:col10/1732036490413/Put/seqid=0 2024-11-19T17:14:53,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742151_1327 (size=12241) 2024-11-19T17:14:53,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742152_1328 (size=12241) 2024-11-19T17:14:53,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T17:14:53,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:53,920 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing c1ce7345860509b228081c672a073b0d 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-19T17:14:53,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=A 2024-11-19T17:14:53,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:53,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=B 2024-11-19T17:14:53,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:53,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=C 2024-11-19T17:14:53,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:53,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/1ff8553674c14a0c8789f77c41f4cd4e is 50, key is test_row_0/A:col10/1732036492670/Put/seqid=0 2024-11-19T17:14:53,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742153_1329 (size=12101) 2024-11-19T17:14:54,269 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/2d98932586504a0c8ebe21b513666bd7 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/2d98932586504a0c8ebe21b513666bd7 2024-11-19T17:14:54,269 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/e575b39b5e6f43569445a8fb9bc1a933 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/e575b39b5e6f43569445a8fb9bc1a933 2024-11-19T17:14:54,275 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1ce7345860509b228081c672a073b0d/A of c1ce7345860509b228081c672a073b0d into 2d98932586504a0c8ebe21b513666bd7(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:54,275 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1ce7345860509b228081c672a073b0d/B of c1ce7345860509b228081c672a073b0d into e575b39b5e6f43569445a8fb9bc1a933(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:54,275 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:14:54,275 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:14:54,275 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., storeName=c1ce7345860509b228081c672a073b0d/B, priority=12, startTime=1732036493847; duration=0sec 2024-11-19T17:14:54,275 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., storeName=c1ce7345860509b228081c672a073b0d/A, priority=12, startTime=1732036493847; duration=0sec 2024-11-19T17:14:54,276 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:54,276 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1ce7345860509b228081c672a073b0d:B 2024-11-19T17:14:54,276 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:54,276 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1ce7345860509b228081c672a073b0d:A 2024-11-19T17:14:54,276 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:14:54,277 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:14:54,277 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): c1ce7345860509b228081c672a073b0d/C is initiating minor compaction (all files) 2024-11-19T17:14:54,277 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1ce7345860509b228081c672a073b0d/C in TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:54,277 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/49d9d1f76f664682a238f4cc502ca439, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/9c398c079d0247a3842a6462d14d8b4c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/0d1da9f998c74dcbae934a1d5bd5584b, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/f47db503b7674b2d8e95aa1ecdcc05d4] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp, totalSize=44.7 K 2024-11-19T17:14:54,278 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 49d9d1f76f664682a238f4cc502ca439, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732036488591 2024-11-19T17:14:54,278 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c398c079d0247a3842a6462d14d8b4c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1732036490396 2024-11-19T17:14:54,278 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d1da9f998c74dcbae934a1d5bd5584b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732036490413 2024-11-19T17:14:54,279 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting f47db503b7674b2d8e95aa1ecdcc05d4, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732036492585 2024-11-19T17:14:54,293 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1ce7345860509b228081c672a073b0d#C#compaction#277 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:54,294 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/75f7b6328e9e422692e0754acc5470a6 is 50, key is test_row_0/C:col10/1732036490413/Put/seqid=0 2024-11-19T17:14:54,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742154_1330 (size=12241) 2024-11-19T17:14:54,302 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/75f7b6328e9e422692e0754acc5470a6 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/75f7b6328e9e422692e0754acc5470a6 2024-11-19T17:14:54,307 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1ce7345860509b228081c672a073b0d/C of c1ce7345860509b228081c672a073b0d into 75f7b6328e9e422692e0754acc5470a6(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:54,307 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:14:54,307 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., storeName=c1ce7345860509b228081c672a073b0d/C, priority=12, startTime=1732036493847; duration=0sec 2024-11-19T17:14:54,307 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:54,307 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1ce7345860509b228081c672a073b0d:C 2024-11-19T17:14:54,329 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/1ff8553674c14a0c8789f77c41f4cd4e 2024-11-19T17:14:54,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/b885e635aabf48ea92e5c82bb89bdaad is 50, key is test_row_0/B:col10/1732036492670/Put/seqid=0 2024-11-19T17:14:54,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742155_1331 (size=12101) 2024-11-19T17:14:54,741 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/b885e635aabf48ea92e5c82bb89bdaad 2024-11-19T17:14:54,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/50693a1c58b64cb79ecf4c79cdb95a48 is 50, key is test_row_0/C:col10/1732036492670/Put/seqid=0 2024-11-19T17:14:54,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742156_1332 (size=12101) 2024-11-19T17:14:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on c1ce7345860509b228081c672a073b0d 2024-11-19T17:14:54,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:14:54,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036554793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:54,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:54,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:54,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:54,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036554796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:54,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:54,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:54,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:54,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036554824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:54,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036554824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:54,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036554824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:54,826 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_2' in c1ce7345860509b228081c672a073b0d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:14:54,826 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_0' in c1ce7345860509b228081c672a073b0d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:14:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:54,826 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_2' in c1ce7345860509b228081c672a073b0d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:14:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T17:14:54,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036554903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:54,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:54,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:54,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:54,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:54,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036554914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:54,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:54,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:54,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:55,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036555113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:55,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:55,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:55,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:55,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:55,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036555123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:55,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:55,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:55,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:55,152 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/50693a1c58b64cb79ecf4c79cdb95a48 2024-11-19T17:14:55,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/1ff8553674c14a0c8789f77c41f4cd4e as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/1ff8553674c14a0c8789f77c41f4cd4e 2024-11-19T17:14:55,165 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/1ff8553674c14a0c8789f77c41f4cd4e, entries=150, sequenceid=131, filesize=11.8 K 2024-11-19T17:14:55,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/b885e635aabf48ea92e5c82bb89bdaad as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/b885e635aabf48ea92e5c82bb89bdaad 2024-11-19T17:14:55,170 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/b885e635aabf48ea92e5c82bb89bdaad, entries=150, sequenceid=131, filesize=11.8 K 2024-11-19T17:14:55,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/50693a1c58b64cb79ecf4c79cdb95a48 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/50693a1c58b64cb79ecf4c79cdb95a48 2024-11-19T17:14:55,174 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/50693a1c58b64cb79ecf4c79cdb95a48, entries=150, sequenceid=131, filesize=11.8 K 2024-11-19T17:14:55,175 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for c1ce7345860509b228081c672a073b0d in 1255ms, sequenceid=131, compaction requested=false 2024-11-19T17:14:55,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:14:55,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:55,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-19T17:14:55,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-19T17:14:55,179 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-19T17:14:55,179 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4800 sec 2024-11-19T17:14:55,181 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.4850 sec 2024-11-19T17:14:55,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on c1ce7345860509b228081c672a073b0d 2024-11-19T17:14:55,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1ce7345860509b228081c672a073b0d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T17:14:55,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=A 2024-11-19T17:14:55,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:55,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=B 2024-11-19T17:14:55,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:55,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=C 2024-11-19T17:14:55,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:55,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/a635c9962c344bb78da80daca4d41631 is 50, key is test_row_0/A:col10/1732036495426/Put/seqid=0 2024-11-19T17:14:55,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742157_1333 (size=14541) 2024-11-19T17:14:55,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036555473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:55,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:55,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:55,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:55,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036555479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:55,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036555583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:55,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:55,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:55,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036555593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:55,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:55,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:55,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:55,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036555794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:55,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:55,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:55,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:55,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036555803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:55,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:55,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:55,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:55,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/a635c9962c344bb78da80daca4d41631 2024-11-19T17:14:55,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/cf8ec834b57d40c2b93a2259233d5ea9 is 50, key is test_row_0/B:col10/1732036495426/Put/seqid=0 2024-11-19T17:14:55,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742158_1334 (size=12151) 2024-11-19T17:14:56,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:56,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036556103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:56,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036556126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:56,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/cf8ec834b57d40c2b93a2259233d5ea9 2024-11-19T17:14:56,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/afef44b4499a471e99dcf2126230e470 is 50, key is test_row_0/C:col10/1732036495426/Put/seqid=0 2024-11-19T17:14:56,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742159_1335 (size=12151) 2024-11-19T17:14:56,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:56,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036556613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:56,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:56,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036556644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:56,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:56,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:56,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:56,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/afef44b4499a471e99dcf2126230e470 2024-11-19T17:14:56,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/a635c9962c344bb78da80daca4d41631 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/a635c9962c344bb78da80daca4d41631 2024-11-19T17:14:56,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/a635c9962c344bb78da80daca4d41631, entries=200, sequenceid=146, filesize=14.2 K 2024-11-19T17:14:56,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/cf8ec834b57d40c2b93a2259233d5ea9 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/cf8ec834b57d40c2b93a2259233d5ea9 2024-11-19T17:14:56,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/cf8ec834b57d40c2b93a2259233d5ea9, entries=150, sequenceid=146, filesize=11.9 K 2024-11-19T17:14:56,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/afef44b4499a471e99dcf2126230e470 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/afef44b4499a471e99dcf2126230e470 2024-11-19T17:14:56,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/afef44b4499a471e99dcf2126230e470, entries=150, sequenceid=146, filesize=11.9 K 2024-11-19T17:14:56,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c1ce7345860509b228081c672a073b0d in 1258ms, sequenceid=146, compaction requested=true 2024-11-19T17:14:56,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:14:56,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1ce7345860509b228081c672a073b0d:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:14:56,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:56,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1ce7345860509b228081c672a073b0d:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:14:56,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:56,686 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:56,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1ce7345860509b228081c672a073b0d:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:14:56,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:56,686 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:56,687 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38883 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:56,687 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): c1ce7345860509b228081c672a073b0d/A is initiating minor compaction (all files) 2024-11-19T17:14:56,687 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1ce7345860509b228081c672a073b0d/A in TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:56,687 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/2d98932586504a0c8ebe21b513666bd7, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/1ff8553674c14a0c8789f77c41f4cd4e, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/a635c9962c344bb78da80daca4d41631] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp, totalSize=38.0 K 2024-11-19T17:14:56,687 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:56,687 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): c1ce7345860509b228081c672a073b0d/B is initiating minor compaction (all files) 2024-11-19T17:14:56,688 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1ce7345860509b228081c672a073b0d/B in TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:56,688 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/e575b39b5e6f43569445a8fb9bc1a933, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/b885e635aabf48ea92e5c82bb89bdaad, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/cf8ec834b57d40c2b93a2259233d5ea9] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp, totalSize=35.6 K 2024-11-19T17:14:56,688 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d98932586504a0c8ebe21b513666bd7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732036490413 2024-11-19T17:14:56,688 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting e575b39b5e6f43569445a8fb9bc1a933, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732036490413 2024-11-19T17:14:56,688 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ff8553674c14a0c8789f77c41f4cd4e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732036492662 2024-11-19T17:14:56,688 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting b885e635aabf48ea92e5c82bb89bdaad, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732036492662 2024-11-19T17:14:56,689 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting a635c9962c344bb78da80daca4d41631, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732036494788 2024-11-19T17:14:56,689 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting cf8ec834b57d40c2b93a2259233d5ea9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732036494791 2024-11-19T17:14:56,698 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1ce7345860509b228081c672a073b0d#B#compaction#283 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:56,698 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1ce7345860509b228081c672a073b0d#A#compaction#284 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:56,698 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/12d540fdb7b344448458f5572423795d is 50, key is test_row_0/A:col10/1732036495426/Put/seqid=0 2024-11-19T17:14:56,698 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/07cedf54c63d4f0688ec3db1fc8f95a9 is 50, key is test_row_0/B:col10/1732036495426/Put/seqid=0 2024-11-19T17:14:56,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742161_1337 (size=12493) 2024-11-19T17:14:56,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742160_1336 (size=12493) 2024-11-19T17:14:56,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on c1ce7345860509b228081c672a073b0d 2024-11-19T17:14:56,836 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1ce7345860509b228081c672a073b0d 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-19T17:14:56,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=A 2024-11-19T17:14:56,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:56,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=B 2024-11-19T17:14:56,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:56,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=C 2024-11-19T17:14:56,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:56,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/da551f0793eb4cabae3c9981dd680b66 is 50, key is test_row_0/A:col10/1732036496835/Put/seqid=0 2024-11-19T17:14:56,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742162_1338 (size=12151) 2024-11-19T17:14:56,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T17:14:56,854 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:14:56,855 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:14:56,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-19T17:14:56,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T17:14:56,856 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:14:56,857 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:14:56,857 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:14:56,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:56,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:56,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036556854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:56,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036556854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:56,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:56,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:56,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:56,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:56,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:56,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:56,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:56,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036556858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:56,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:56,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:56,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T17:14:56,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036556964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:56,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036556964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:56,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036556964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:57,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-19T17:14:57,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:57,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:14:57,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:57,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:57,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:57,117 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/12d540fdb7b344448458f5572423795d as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/12d540fdb7b344448458f5572423795d 2024-11-19T17:14:57,119 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/07cedf54c63d4f0688ec3db1fc8f95a9 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/07cedf54c63d4f0688ec3db1fc8f95a9 2024-11-19T17:14:57,122 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1ce7345860509b228081c672a073b0d/A of c1ce7345860509b228081c672a073b0d into 12d540fdb7b344448458f5572423795d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:57,122 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:14:57,122 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., storeName=c1ce7345860509b228081c672a073b0d/A, priority=13, startTime=1732036496686; duration=0sec 2024-11-19T17:14:57,122 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:57,122 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1ce7345860509b228081c672a073b0d:A 2024-11-19T17:14:57,122 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:14:57,123 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1ce7345860509b228081c672a073b0d/B of c1ce7345860509b228081c672a073b0d into 07cedf54c63d4f0688ec3db1fc8f95a9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:57,123 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:14:57,123 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., storeName=c1ce7345860509b228081c672a073b0d/B, priority=13, startTime=1732036496686; duration=0sec 2024-11-19T17:14:57,124 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:57,124 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1ce7345860509b228081c672a073b0d:B 2024-11-19T17:14:57,124 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:14:57,124 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): c1ce7345860509b228081c672a073b0d/C is initiating minor compaction (all files) 2024-11-19T17:14:57,124 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1ce7345860509b228081c672a073b0d/C in TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:57,124 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/75f7b6328e9e422692e0754acc5470a6, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/50693a1c58b64cb79ecf4c79cdb95a48, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/afef44b4499a471e99dcf2126230e470] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp, totalSize=35.6 K 2024-11-19T17:14:57,124 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75f7b6328e9e422692e0754acc5470a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732036490413 2024-11-19T17:14:57,125 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50693a1c58b64cb79ecf4c79cdb95a48, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732036492662 2024-11-19T17:14:57,125 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting afef44b4499a471e99dcf2126230e470, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732036494791 2024-11-19T17:14:57,131 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1ce7345860509b228081c672a073b0d#C#compaction#286 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:57,132 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/4758be2ed4ab4e109f79b7be0d695f1f is 50, key is test_row_0/C:col10/1732036495426/Put/seqid=0 2024-11-19T17:14:57,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742163_1339 (size=12493) 2024-11-19T17:14:57,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-19T17:14:57,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:57,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:14:57,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:57,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:57,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T17:14:57,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036557174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:57,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:57,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:57,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:57,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036557174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:57,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036557174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:57,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:57,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:57,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:57,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:57,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:57,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:57,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/da551f0793eb4cabae3c9981dd680b66 2024-11-19T17:14:57,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/60bc273fe368443db990ce87e538f063 is 50, key is test_row_0/B:col10/1732036496835/Put/seqid=0 2024-11-19T17:14:57,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742164_1340 (size=12151) 2024-11-19T17:14:57,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-19T17:14:57,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:57,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:14:57,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:57,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:57,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:57,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-19T17:14:57,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:57,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:14:57,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:57,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:57,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:57,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:57,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T17:14:57,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036557484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:57,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:57,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036557485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:57,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:57,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:57,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:57,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:57,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036557485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:57,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:57,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:57,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:57,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:57,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:57,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:57,559 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/4758be2ed4ab4e109f79b7be0d695f1f as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/4758be2ed4ab4e109f79b7be0d695f1f 2024-11-19T17:14:57,565 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1ce7345860509b228081c672a073b0d/C of c1ce7345860509b228081c672a073b0d into 4758be2ed4ab4e109f79b7be0d695f1f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:14:57,565 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:14:57,565 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., storeName=c1ce7345860509b228081c672a073b0d/C, priority=13, startTime=1732036496686; duration=0sec 2024-11-19T17:14:57,565 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:57,565 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1ce7345860509b228081c672a073b0d:C 2024-11-19T17:14:57,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-19T17:14:57,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:57,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:14:57,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:57,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:57,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:57,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:57,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036557624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:57,628 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_1' in c1ce7345860509b228081c672a073b0d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:14:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:57,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:57,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036557664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:57,667 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_1' in c1ce7345860509b228081c672a073b0d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:14:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:57,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/60bc273fe368443db990ce87e538f063 2024-11-19T17:14:57,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/50d9012ef91c498f97b681156328718c is 50, key is test_row_0/C:col10/1732036496835/Put/seqid=0 2024-11-19T17:14:57,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742165_1341 (size=12151) 2024-11-19T17:14:57,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/50d9012ef91c498f97b681156328718c 2024-11-19T17:14:57,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/da551f0793eb4cabae3c9981dd680b66 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/da551f0793eb4cabae3c9981dd680b66 2024-11-19T17:14:57,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/da551f0793eb4cabae3c9981dd680b66, entries=150, sequenceid=167, filesize=11.9 K 2024-11-19T17:14:57,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/60bc273fe368443db990ce87e538f063 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/60bc273fe368443db990ce87e538f063 2024-11-19T17:14:57,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/60bc273fe368443db990ce87e538f063, entries=150, sequenceid=167, filesize=11.9 K 2024-11-19T17:14:57,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/50d9012ef91c498f97b681156328718c as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/50d9012ef91c498f97b681156328718c 2024-11-19T17:14:57,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/50d9012ef91c498f97b681156328718c, entries=150, sequenceid=167, filesize=11.9 K 2024-11-19T17:14:57,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for c1ce7345860509b228081c672a073b0d in 918ms, sequenceid=167, compaction requested=false 2024-11-19T17:14:57,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:14:57,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-19T17:14:57,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:57,778 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing c1ce7345860509b228081c672a073b0d 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T17:14:57,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=A 2024-11-19T17:14:57,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:57,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=B 2024-11-19T17:14:57,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:57,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=C 2024-11-19T17:14:57,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:57,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/c767fecdc05b4f22a30f7abf86c03dee is 50, key is test_row_0/A:col10/1732036496857/Put/seqid=0 2024-11-19T17:14:57,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742166_1342 (size=12151) 2024-11-19T17:14:57,797 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/c767fecdc05b4f22a30f7abf86c03dee 2024-11-19T17:14:57,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/7c3b14ddaade419eb3d3b614d900ec48 is 50, key is test_row_0/B:col10/1732036496857/Put/seqid=0 2024-11-19T17:14:57,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742167_1343 (size=12151) 2024-11-19T17:14:57,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T17:14:57,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on c1ce7345860509b228081c672a073b0d 2024-11-19T17:14:57,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:14:58,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:58,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036558026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:58,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:58,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036558027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:58,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036498010, max timestamp=1732036498030Num uniques 859; 2024-11-19T17:14:58,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:14:58,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036498010, max timestamp=1732036498030Num uniques 859; 2024-11-19T17:14:58,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:14:58,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:58,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:58,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:58,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036498010, max timestamp=1732036498030Num uniques 859; 2024-11-19T17:14:58,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:14:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:58,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:58,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036558137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:58,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:58,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:58,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:58,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:58,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036558144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:58,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:58,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:58,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:58,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036558144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:58,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:58,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:58,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:58,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:58,217 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/7c3b14ddaade419eb3d3b614d900ec48 2024-11-19T17:14:58,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/1d7dbb9fd8bb442e92c61e9ea3c775ca is 50, key is test_row_0/C:col10/1732036496857/Put/seqid=0 2024-11-19T17:14:58,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742168_1344 (size=12151) 2024-11-19T17:14:58,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:58,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036558254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:58,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:58,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036558353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:58,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:58,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:58,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:58,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:58,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036558353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:58,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:58,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:58,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:58,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:58,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036558463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:58,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:58,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:58,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:58,631 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/1d7dbb9fd8bb442e92c61e9ea3c775ca 2024-11-19T17:14:58,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/c767fecdc05b4f22a30f7abf86c03dee as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/c767fecdc05b4f22a30f7abf86c03dee 2024-11-19T17:14:58,639 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/c767fecdc05b4f22a30f7abf86c03dee, entries=150, sequenceid=181, filesize=11.9 K 2024-11-19T17:14:58,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/7c3b14ddaade419eb3d3b614d900ec48 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/7c3b14ddaade419eb3d3b614d900ec48 2024-11-19T17:14:58,644 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/7c3b14ddaade419eb3d3b614d900ec48, entries=150, sequenceid=181, filesize=11.9 K 2024-11-19T17:14:58,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/1d7dbb9fd8bb442e92c61e9ea3c775ca as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/1d7dbb9fd8bb442e92c61e9ea3c775ca 2024-11-19T17:14:58,649 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/1d7dbb9fd8bb442e92c61e9ea3c775ca, entries=150, sequenceid=181, filesize=11.9 K 2024-11-19T17:14:58,650 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for c1ce7345860509b228081c672a073b0d in 872ms, sequenceid=181, compaction requested=true 2024-11-19T17:14:58,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:14:58,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:58,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-19T17:14:58,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-19T17:14:58,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-19T17:14:58,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7940 sec 2024-11-19T17:14:58,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.7990 sec 2024-11-19T17:14:58,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on c1ce7345860509b228081c672a073b0d 2024-11-19T17:14:58,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1ce7345860509b228081c672a073b0d 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-19T17:14:58,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=A 2024-11-19T17:14:58,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:58,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=B 2024-11-19T17:14:58,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:58,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=C 2024-11-19T17:14:58,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:14:58,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/4fc232dc5001497dbc9c36f309d33d17 is 50, key is test_row_0/A:col10/1732036498099/Put/seqid=0 2024-11-19T17:14:58,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742169_1345 (size=14541) 2024-11-19T17:14:58,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:58,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036558688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:58,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:58,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:58,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:58,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:58,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036558690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:58,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:58,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:58,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:58,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:58,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036558773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:58,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:58,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:58,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:58,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036558804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:58,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:58,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036558804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:59,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T17:14:59,014 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:14:59,014 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:14:59,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-19T17:14:59,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T17:14:59,016 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:14:59,017 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:14:59,017 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:14:59,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:59,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036559014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:59,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:59,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:59,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:59,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036559014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:59,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:59,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:59,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:59,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:59,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/4fc232dc5001497dbc9c36f309d33d17 2024-11-19T17:14:59,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/9c12e6dcdc6d4dbeb2f1d8d396b794cc is 50, key is test_row_0/B:col10/1732036498099/Put/seqid=0 2024-11-19T17:14:59,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742170_1346 (size=12151) 2024-11-19T17:14:59,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T17:14:59,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-19T17:14:59,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:59,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:14:59,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:59,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:59,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:59,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:59,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:59,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036559284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:59,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:59,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:59,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:59,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-19T17:14:59,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:59,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:14:59,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:59,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:59,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:59,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:59,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:59,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036559323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:59,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:59,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:59,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:59,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036559323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:59,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:59,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:59,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:59,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:59,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T17:14:59,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-19T17:14:59,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:59,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:14:59,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:59,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:59,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:59,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:59,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/9c12e6dcdc6d4dbeb2f1d8d396b794cc 2024-11-19T17:14:59,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/be83d1b679b746e28639269040d78430 is 50, key is test_row_0/C:col10/1732036498099/Put/seqid=0 2024-11-19T17:14:59,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742171_1347 (size=12151) 2024-11-19T17:14:59,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-19T17:14:59,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:59,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:14:59,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:59,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:59,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:59,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:59,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T17:14:59,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:59,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036559643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:59,648 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_1' in c1ce7345860509b228081c672a073b0d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:14:59,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:59,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:59,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:59,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:59,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036559674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:59,679 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_1' in c1ce7345860509b228081c672a073b0d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:14:59,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:59,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:59,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:59,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-19T17:14:59,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:59,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:14:59,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:59,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:59,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:59,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:14:59,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:59,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036559834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:59,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:14:59,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:59,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:59,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036559834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:14:59,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:59,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:59,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:14:59,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:14:59,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/be83d1b679b746e28639269040d78430 2024-11-19T17:14:59,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/4fc232dc5001497dbc9c36f309d33d17 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/4fc232dc5001497dbc9c36f309d33d17 2024-11-19T17:14:59,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/4fc232dc5001497dbc9c36f309d33d17, entries=200, sequenceid=218, filesize=14.2 K 2024-11-19T17:14:59,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/9c12e6dcdc6d4dbeb2f1d8d396b794cc as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/9c12e6dcdc6d4dbeb2f1d8d396b794cc 2024-11-19T17:14:59,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/9c12e6dcdc6d4dbeb2f1d8d396b794cc, entries=150, sequenceid=218, filesize=11.9 K 2024-11-19T17:14:59,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/be83d1b679b746e28639269040d78430 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/be83d1b679b746e28639269040d78430 2024-11-19T17:14:59,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/be83d1b679b746e28639269040d78430, entries=150, sequenceid=218, filesize=11.9 K 2024-11-19T17:14:59,922 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for c1ce7345860509b228081c672a073b0d in 1255ms, sequenceid=218, compaction requested=true 2024-11-19T17:14:59,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:14:59,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1ce7345860509b228081c672a073b0d:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:14:59,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:59,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1ce7345860509b228081c672a073b0d:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:14:59,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:14:59,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1ce7345860509b228081c672a073b0d:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:14:59,922 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:14:59,922 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:14:59,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:14:59,923 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:14:59,923 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51336 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:14:59,923 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): c1ce7345860509b228081c672a073b0d/A is initiating minor compaction (all files) 2024-11-19T17:14:59,923 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): c1ce7345860509b228081c672a073b0d/B is initiating minor compaction (all files) 2024-11-19T17:14:59,923 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1ce7345860509b228081c672a073b0d/A in TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:59,923 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1ce7345860509b228081c672a073b0d/B in TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:59,923 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/07cedf54c63d4f0688ec3db1fc8f95a9, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/60bc273fe368443db990ce87e538f063, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/7c3b14ddaade419eb3d3b614d900ec48, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/9c12e6dcdc6d4dbeb2f1d8d396b794cc] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp, totalSize=47.8 K 2024-11-19T17:14:59,923 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/12d540fdb7b344448458f5572423795d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/da551f0793eb4cabae3c9981dd680b66, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/c767fecdc05b4f22a30f7abf86c03dee, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/4fc232dc5001497dbc9c36f309d33d17] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp, totalSize=50.1 K 2024-11-19T17:14:59,924 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12d540fdb7b344448458f5572423795d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732036494791 2024-11-19T17:14:59,924 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 07cedf54c63d4f0688ec3db1fc8f95a9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732036494791 2024-11-19T17:14:59,924 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting da551f0793eb4cabae3c9981dd680b66, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732036495439 2024-11-19T17:14:59,924 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 60bc273fe368443db990ce87e538f063, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732036495439 2024-11-19T17:14:59,925 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting c767fecdc05b4f22a30f7abf86c03dee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732036496844 2024-11-19T17:14:59,925 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c3b14ddaade419eb3d3b614d900ec48, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732036496844 2024-11-19T17:14:59,925 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fc232dc5001497dbc9c36f309d33d17, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732036498099 2024-11-19T17:14:59,925 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c12e6dcdc6d4dbeb2f1d8d396b794cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732036498099 2024-11-19T17:14:59,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-19T17:14:59,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:14:59,935 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing c1ce7345860509b228081c672a073b0d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T17:14:59,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=A 2024-11-19T17:14:59,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:59,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=B 2024-11-19T17:14:59,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:59,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=C 2024-11-19T17:14:59,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:14:59,938 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1ce7345860509b228081c672a073b0d#B#compaction#295 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:59,938 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/14e8700997ce45d3a2823bcaaccefb40 is 50, key is test_row_0/B:col10/1732036498099/Put/seqid=0 2024-11-19T17:14:59,939 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1ce7345860509b228081c672a073b0d#A#compaction#296 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:14:59,940 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/0234dd82a57d41419c24b048a2bebc27 is 50, key is test_row_0/A:col10/1732036498099/Put/seqid=0 2024-11-19T17:14:59,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/c7364cb6539b4ee98c47a8e6ab064f1a is 50, key is test_row_0/A:col10/1732036498680/Put/seqid=0 2024-11-19T17:14:59,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742172_1348 (size=12629) 2024-11-19T17:14:59,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742173_1349 (size=12629) 2024-11-19T17:14:59,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742174_1350 (size=12151) 2024-11-19T17:14:59,971 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/c7364cb6539b4ee98c47a8e6ab064f1a 2024-11-19T17:14:59,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/eec6d8189e0440eba1e179404e397bbd is 50, key is test_row_0/B:col10/1732036498680/Put/seqid=0 2024-11-19T17:14:59,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742175_1351 (size=12151) 2024-11-19T17:15:00,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T17:15:00,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on c1ce7345860509b228081c672a073b0d 2024-11-19T17:15:00,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:15:00,365 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/14e8700997ce45d3a2823bcaaccefb40 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/14e8700997ce45d3a2823bcaaccefb40 2024-11-19T17:15:00,365 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/0234dd82a57d41419c24b048a2bebc27 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/0234dd82a57d41419c24b048a2bebc27 2024-11-19T17:15:00,370 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1ce7345860509b228081c672a073b0d/B of c1ce7345860509b228081c672a073b0d into 14e8700997ce45d3a2823bcaaccefb40(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:00,370 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:15:00,370 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1ce7345860509b228081c672a073b0d/A of c1ce7345860509b228081c672a073b0d into 0234dd82a57d41419c24b048a2bebc27(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:00,370 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., storeName=c1ce7345860509b228081c672a073b0d/B, priority=12, startTime=1732036499922; duration=0sec 2024-11-19T17:15:00,370 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:15:00,370 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., storeName=c1ce7345860509b228081c672a073b0d/A, priority=12, startTime=1732036499922; duration=0sec 2024-11-19T17:15:00,370 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:00,370 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1ce7345860509b228081c672a073b0d:B 2024-11-19T17:15:00,370 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:15:00,370 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:00,370 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1ce7345860509b228081c672a073b0d:A 2024-11-19T17:15:00,372 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:15:00,372 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): c1ce7345860509b228081c672a073b0d/C is initiating minor compaction (all files) 2024-11-19T17:15:00,372 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1ce7345860509b228081c672a073b0d/C in TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:00,372 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/4758be2ed4ab4e109f79b7be0d695f1f, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/50d9012ef91c498f97b681156328718c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/1d7dbb9fd8bb442e92c61e9ea3c775ca, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/be83d1b679b746e28639269040d78430] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp, totalSize=47.8 K 2024-11-19T17:15:00,372 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 4758be2ed4ab4e109f79b7be0d695f1f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732036494791 2024-11-19T17:15:00,373 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 50d9012ef91c498f97b681156328718c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732036495439 2024-11-19T17:15:00,373 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d7dbb9fd8bb442e92c61e9ea3c775ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732036496844 2024-11-19T17:15:00,373 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting be83d1b679b746e28639269040d78430, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732036498099 2024-11-19T17:15:00,381 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/eec6d8189e0440eba1e179404e397bbd 2024-11-19T17:15:00,384 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1ce7345860509b228081c672a073b0d#C#compaction#299 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:00,385 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/1430bc0cec6c4025b812c843d05f0b09 is 50, key is test_row_0/C:col10/1732036498099/Put/seqid=0 2024-11-19T17:15:00,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036500366, max timestamp=1732036500395Num uniques 859; 2024-11-19T17:15:00,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036500366, max timestamp=1732036500395Num uniques 859; 2024-11-19T17:15:00,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:15:00,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036500366, max timestamp=1732036500395Num uniques 859; 2024-11-19T17:15:00,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:15:00,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:15:00,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/8f03d7f0711d43bab4168b9701ed6a26 is 50, key is test_row_0/C:col10/1732036498680/Put/seqid=0 2024-11-19T17:15:00,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742176_1352 (size=12629) 2024-11-19T17:15:00,418 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/1430bc0cec6c4025b812c843d05f0b09 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/1430bc0cec6c4025b812c843d05f0b09 2024-11-19T17:15:00,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742177_1353 (size=12151) 2024-11-19T17:15:00,429 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1ce7345860509b228081c672a073b0d/C of c1ce7345860509b228081c672a073b0d into 1430bc0cec6c4025b812c843d05f0b09(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:00,429 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:15:00,429 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., storeName=c1ce7345860509b228081c672a073b0d/C, priority=12, startTime=1732036499922; duration=0sec 2024-11-19T17:15:00,429 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:00,429 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1ce7345860509b228081c672a073b0d:C 2024-11-19T17:15:00,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:00,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036560504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:00,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:00,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:00,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:00,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:00,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036560614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:00,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:00,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:00,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:00,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:00,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036560823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:00,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:00,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:00,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:00,829 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/8f03d7f0711d43bab4168b9701ed6a26 2024-11-19T17:15:00,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/c7364cb6539b4ee98c47a8e6ab064f1a as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/c7364cb6539b4ee98c47a8e6ab064f1a 2024-11-19T17:15:00,837 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/c7364cb6539b4ee98c47a8e6ab064f1a, entries=150, sequenceid=228, filesize=11.9 K 2024-11-19T17:15:00,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/eec6d8189e0440eba1e179404e397bbd as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/eec6d8189e0440eba1e179404e397bbd 2024-11-19T17:15:00,841 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/eec6d8189e0440eba1e179404e397bbd, entries=150, sequenceid=228, filesize=11.9 K 2024-11-19T17:15:00,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/8f03d7f0711d43bab4168b9701ed6a26 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/8f03d7f0711d43bab4168b9701ed6a26 2024-11-19T17:15:00,846 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/8f03d7f0711d43bab4168b9701ed6a26, entries=150, sequenceid=228, filesize=11.9 K 2024-11-19T17:15:00,847 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c1ce7345860509b228081c672a073b0d in 912ms, sequenceid=228, compaction requested=false 2024-11-19T17:15:00,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:15:00,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:00,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-19T17:15:00,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-19T17:15:00,851 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-19T17:15:00,851 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8320 sec 2024-11-19T17:15:00,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.8370 sec 2024-11-19T17:15:00,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on c1ce7345860509b228081c672a073b0d 2024-11-19T17:15:00,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1ce7345860509b228081c672a073b0d 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-19T17:15:00,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732036500492, max timestamp=1732036500856Num uniques 1018; 2024-11-19T17:15:00,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732036500492, max timestamp=1732036500856Num uniques 1018; 2024-11-19T17:15:00,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-19T17:15:00,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-19T17:15:00,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732036500492, max timestamp=1732036500856Num uniques 1018; 2024-11-19T17:15:00,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-19T17:15:00,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=A 2024-11-19T17:15:00,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:00,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=B 2024-11-19T17:15:00,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:00,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=C 2024-11-19T17:15:00,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:00,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/e86e292eaeca40bf954202053e01b78e is 50, key is test_row_0/A:col10/1732036500856/Put/seqid=0 2024-11-19T17:15:00,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742178_1354 (size=14741) 2024-11-19T17:15:00,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/e86e292eaeca40bf954202053e01b78e 2024-11-19T17:15:00,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/3395f0bc41ec4d86b9d8a61423a02d6d is 50, key is test_row_0/B:col10/1732036500856/Put/seqid=0 2024-11-19T17:15:00,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742179_1355 (size=12301) 2024-11-19T17:15:00,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036500892, max timestamp=1732036500916Num uniques 859; 2024-11-19T17:15:00,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:15:00,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036500892, max timestamp=1732036500916Num uniques 859; 2024-11-19T17:15:00,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:15:00,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036500892, max timestamp=1732036500916Num uniques 859; 2024-11-19T17:15:00,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:15:00,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732036500961, max timestamp=1732036500975Num uniques 1068; 2024-11-19T17:15:00,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-19T17:15:00,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732036500961, max timestamp=1732036500975Num uniques 1068; 2024-11-19T17:15:00,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-19T17:15:00,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732036500961, max timestamp=1732036500975Num uniques 1018; 2024-11-19T17:15:00,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-19T17:15:01,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732036501026, max timestamp=1732036501039Num uniques 1027; 2024-11-19T17:15:01,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732036501026, max timestamp=1732036501039Num uniques 1027; 2024-11-19T17:15:01,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-19T17:15:01,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-19T17:15:01,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732036501026, max timestamp=1732036501039Num uniques 1027; 2024-11-19T17:15:01,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-19T17:15:01,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732036501078, max timestamp=1732036501102Num uniques 1036; 2024-11-19T17:15:01,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-19T17:15:01,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732036501078, max timestamp=1732036501102Num uniques 1036; 2024-11-19T17:15:01,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732036501078, max timestamp=1732036501102Num uniques 1036; 2024-11-19T17:15:01,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-19T17:15:01,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-19T17:15:01,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732036501139, max timestamp=1732036501155Num uniques 1095; 2024-11-19T17:15:01,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732036501139, max timestamp=1732036501155Num uniques 1095; 2024-11-19T17:15:01,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-19T17:15:01,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-19T17:15:01,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732036501139, max timestamp=1732036501155Num uniques 1095; 2024-11-19T17:15:01,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-19T17:15:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T17:15:01,164 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:15:01,164 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:15:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-19T17:15:01,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T17:15:01,166 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:15:01,167 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:15:01,167 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:15:01,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:01,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036561196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:01,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:01,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:01,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:01,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036561198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:01,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:01,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:01,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:01,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732036501178, max timestamp=1732036501200Num uniques 1104; 2024-11-19T17:15:01,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732036501178, max timestamp=1732036501200Num uniques 1104; 2024-11-19T17:15:01,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732036501178, max timestamp=1732036501200Num uniques 1104; 2024-11-19T17:15:01,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-19T17:15:01,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-19T17:15:01,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-19T17:15:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T17:15:01,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/3395f0bc41ec4d86b9d8a61423a02d6d 2024-11-19T17:15:01,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/3ba9c918259046a7be5faeda129904e7 is 50, key is test_row_0/C:col10/1732036500856/Put/seqid=0 2024-11-19T17:15:01,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742180_1356 (size=12301) 2024-11-19T17:15:01,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732036501284, max timestamp=1732036501309Num uniques 1063; 2024-11-19T17:15:01,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732036501284, max timestamp=1732036501309Num uniques 1063; 2024-11-19T17:15:01,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-19T17:15:01,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-19T17:15:01,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732036501284, max timestamp=1732036501309Num uniques 1063; 2024-11-19T17:15:01,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-19T17:15:01,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T17:15:01,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:01,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:15:01,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:01,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:01,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:01,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:01,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1732036501345, max timestamp=1732036501357Num uniques 1072; 2024-11-19T17:15:01,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1732036501345, max timestamp=1732036501357Num uniques 1072; 2024-11-19T17:15:01,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-19T17:15:01,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-19T17:15:01,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1732036501345, max timestamp=1732036501357Num uniques 1072; 2024-11-19T17:15:01,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-19T17:15:01,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732036501382, max timestamp=1732036501399Num uniques 1081; 2024-11-19T17:15:01,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-19T17:15:01,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732036501382, max timestamp=1732036501399Num uniques 1081; 2024-11-19T17:15:01,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-19T17:15:01,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1732036501382, max timestamp=1732036501399Num uniques 1081; 2024-11-19T17:15:01,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-19T17:15:01,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1732036501423, max timestamp=1732036501446Num uniques 1090; 2024-11-19T17:15:01,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=13280, suffix heap size=141328,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=290 2024-11-19T17:15:01,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732036501423, max timestamp=1732036501446Num uniques 1040; 2024-11-19T17:15:01,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1732036501423, max timestamp=1732036501446Num uniques 1090; 2024-11-19T17:15:01,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=6870, suffix heap size=136128,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=150 2024-11-19T17:15:01,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=13280, suffix heap size=141328,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=290 2024-11-19T17:15:01,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T17:15:01,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:01,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:15:01,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:01,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:01,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:01,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T17:15:01,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:01,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036561506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:01,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:01,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:01,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:01,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1732036501498, max timestamp=1732036501508Num uniques 1149; 2024-11-19T17:15:01,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=11404, suffix heap size=147464,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=249 2024-11-19T17:15:01,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1732036501498, max timestamp=1732036501508Num uniques 1009; 2024-11-19T17:15:01,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=11404, suffix heap size=132904,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=249 2024-11-19T17:15:01,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1732036501498, max timestamp=1732036501508Num uniques 1149; 2024-11-19T17:15:01,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=11404, suffix heap size=147464,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=249 2024-11-19T17:15:01,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:01,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036561573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:01,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:01,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:01,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:01,582 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1732036501555, max timestamp=1732036501577Num uniques 1108; 2024-11-19T17:15:01,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=7230, suffix heap size=143200,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=158 2024-11-19T17:15:01,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1732036501555, max timestamp=1732036501577Num uniques 1108; 2024-11-19T17:15:01,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=7230, suffix heap size=143200,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=158 2024-11-19T17:15:01,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1732036501555, max timestamp=1732036501577Num uniques 1108; 2024-11-19T17:15:01,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=7230, suffix heap size=143200,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=158 2024-11-19T17:15:01,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T17:15:01,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:01,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:15:01,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:01,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:01,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:01,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1732036501635, max timestamp=1732036501656Num uniques 1017; 2024-11-19T17:15:01,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=7642, suffix heap size=133736,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=167 2024-11-19T17:15:01,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1732036501635, max timestamp=1732036501656Num uniques 1017; 2024-11-19T17:15:01,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=7642, suffix heap size=133736,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=167 2024-11-19T17:15:01,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1732036501635, max timestamp=1732036501656Num uniques 1017; 2024-11-19T17:15:01,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=7642, suffix heap size=133736,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=167 2024-11-19T17:15:01,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/3ba9c918259046a7be5faeda129904e7 2024-11-19T17:15:01,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1732036501690, max timestamp=1732036501700Num uniques 1026; 2024-11-19T17:15:01,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-11-19T17:15:01,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1732036501690, max timestamp=1732036501700Num uniques 1026; 2024-11-19T17:15:01,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-11-19T17:15:01,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1732036501690, max timestamp=1732036501700Num uniques 1026; 2024-11-19T17:15:01,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-11-19T17:15:01,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/e86e292eaeca40bf954202053e01b78e as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/e86e292eaeca40bf954202053e01b78e 2024-11-19T17:15:01,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/e86e292eaeca40bf954202053e01b78e, entries=200, sequenceid=270, filesize=14.4 K 2024-11-19T17:15:01,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/3395f0bc41ec4d86b9d8a61423a02d6d as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/3395f0bc41ec4d86b9d8a61423a02d6d 2024-11-19T17:15:01,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/3395f0bc41ec4d86b9d8a61423a02d6d, entries=150, sequenceid=270, filesize=12.0 K 2024-11-19T17:15:01,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/3ba9c918259046a7be5faeda129904e7 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/3ba9c918259046a7be5faeda129904e7 2024-11-19T17:15:01,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/3ba9c918259046a7be5faeda129904e7, entries=150, sequenceid=270, filesize=12.0 K 2024-11-19T17:15:01,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=93.93 KB/96180 for c1ce7345860509b228081c672a073b0d in 873ms, sequenceid=270, compaction requested=true 2024-11-19T17:15:01,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:15:01,732 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:01,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1ce7345860509b228081c672a073b0d:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:15:01,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:01,732 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:01,733 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39521 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:01,733 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): c1ce7345860509b228081c672a073b0d/A is initiating minor compaction (all files) 2024-11-19T17:15:01,733 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1ce7345860509b228081c672a073b0d/A in TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:01,733 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/0234dd82a57d41419c24b048a2bebc27, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/c7364cb6539b4ee98c47a8e6ab064f1a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/e86e292eaeca40bf954202053e01b78e] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp, totalSize=38.6 K 2024-11-19T17:15:01,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1ce7345860509b228081c672a073b0d:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:15:01,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:01,734 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0234dd82a57d41419c24b048a2bebc27, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732036498099 2024-11-19T17:15:01,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1ce7345860509b228081c672a073b0d:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:15:01,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:01,735 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:01,735 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): c1ce7345860509b228081c672a073b0d/B is initiating minor compaction (all files) 2024-11-19T17:15:01,735 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7364cb6539b4ee98c47a8e6ab064f1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732036498673 2024-11-19T17:15:01,735 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1ce7345860509b228081c672a073b0d/B in TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:01,735 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/14e8700997ce45d3a2823bcaaccefb40, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/eec6d8189e0440eba1e179404e397bbd, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/3395f0bc41ec4d86b9d8a61423a02d6d] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp, totalSize=36.2 K 2024-11-19T17:15:01,736 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 14e8700997ce45d3a2823bcaaccefb40, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732036498099 2024-11-19T17:15:01,736 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting e86e292eaeca40bf954202053e01b78e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732036500497 2024-11-19T17:15:01,736 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting eec6d8189e0440eba1e179404e397bbd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732036498673 2024-11-19T17:15:01,737 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 3395f0bc41ec4d86b9d8a61423a02d6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732036500503 2024-11-19T17:15:01,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on c1ce7345860509b228081c672a073b0d 2024-11-19T17:15:01,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1ce7345860509b228081c672a073b0d 3/3 column families, dataSize=100.63 KB heapSize=279.09 KB 2024-11-19T17:15:01,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=A 2024-11-19T17:15:01,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:01,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=B 2024-11-19T17:15:01,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:01,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=C 2024-11-19T17:15:01,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:01,751 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1ce7345860509b228081c672a073b0d#B#compaction#304 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:01,751 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/225d817f2dfa47a39830d80ab39726b5 is 50, key is test_row_0/B:col10/1732036500856/Put/seqid=0 2024-11-19T17:15:01,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/9dd98eab1cec4743aefded6dcd7b8891 is 50, key is test_row_0/A:col10/1732036501738/Put/seqid=0 2024-11-19T17:15:01,755 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1ce7345860509b228081c672a073b0d#A#compaction#306 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:01,756 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/9f738e83a941483db04945354ec3a599 is 50, key is test_row_0/A:col10/1732036500856/Put/seqid=0 2024-11-19T17:15:01,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T17:15:01,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:01,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:01,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036561774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:01,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:15:01,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:01,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:01,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:01,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:01,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:01,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036561778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:01,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:01,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:01,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:01,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:01,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036561779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:01,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:01,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:01,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:01,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742181_1357 (size=12881) 2024-11-19T17:15:01,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T17:15:01,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742182_1358 (size=12301) 2024-11-19T17:15:01,802 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/225d817f2dfa47a39830d80ab39726b5 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/225d817f2dfa47a39830d80ab39726b5 2024-11-19T17:15:01,810 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1ce7345860509b228081c672a073b0d/B of c1ce7345860509b228081c672a073b0d into 225d817f2dfa47a39830d80ab39726b5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:01,810 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:15:01,810 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., storeName=c1ce7345860509b228081c672a073b0d/B, priority=13, startTime=1732036501732; duration=0sec 2024-11-19T17:15:01,810 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:01,810 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1ce7345860509b228081c672a073b0d:B 2024-11-19T17:15:01,810 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:01,813 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:01,813 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): c1ce7345860509b228081c672a073b0d/C is initiating minor compaction (all files) 2024-11-19T17:15:01,813 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1ce7345860509b228081c672a073b0d/C in TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:01,813 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/1430bc0cec6c4025b812c843d05f0b09, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/8f03d7f0711d43bab4168b9701ed6a26, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/3ba9c918259046a7be5faeda129904e7] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp, totalSize=36.2 K 2024-11-19T17:15:01,813 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 1430bc0cec6c4025b812c843d05f0b09, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732036498099 2024-11-19T17:15:01,814 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f03d7f0711d43bab4168b9701ed6a26, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732036498673 2024-11-19T17:15:01,814 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ba9c918259046a7be5faeda129904e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732036500503 2024-11-19T17:15:01,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742183_1359 (size=12881) 2024-11-19T17:15:01,826 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1ce7345860509b228081c672a073b0d#C#compaction#307 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:01,827 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/b45bd95db0644b4bae11ad1732e91c91 is 50, key is test_row_0/C:col10/1732036500856/Put/seqid=0 2024-11-19T17:15:01,828 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/9f738e83a941483db04945354ec3a599 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/9f738e83a941483db04945354ec3a599 2024-11-19T17:15:01,836 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1ce7345860509b228081c672a073b0d/A of c1ce7345860509b228081c672a073b0d into 9f738e83a941483db04945354ec3a599(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:01,836 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:15:01,839 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., storeName=c1ce7345860509b228081c672a073b0d/A, priority=13, startTime=1732036501732; duration=0sec 2024-11-19T17:15:01,839 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:01,839 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1ce7345860509b228081c672a073b0d:A 2024-11-19T17:15:01,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742184_1360 (size=12881) 2024-11-19T17:15:01,869 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/b45bd95db0644b4bae11ad1732e91c91 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/b45bd95db0644b4bae11ad1732e91c91 2024-11-19T17:15:01,876 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1ce7345860509b228081c672a073b0d/C of c1ce7345860509b228081c672a073b0d into b45bd95db0644b4bae11ad1732e91c91(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:01,876 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:15:01,877 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., storeName=c1ce7345860509b228081c672a073b0d/C, priority=13, startTime=1732036501734; duration=0sec 2024-11-19T17:15:01,878 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:01,878 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1ce7345860509b228081c672a073b0d:C 2024-11-19T17:15:01,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:01,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036561883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:01,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:01,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:01,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:01,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:01,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036561894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:01,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:01,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036561894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:01,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:01,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:01,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:01,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:01,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:01,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:01,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T17:15:01,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:01,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:15:01,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:01,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:01,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:02,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T17:15:02,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:02,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:15:02,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:02,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:02,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:02,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:02,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036562094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:02,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:02,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:02,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:02,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:02,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036562103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:02,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:02,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:02,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:02,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036562103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:02,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:02,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:02,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:02,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:02,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/9dd98eab1cec4743aefded6dcd7b8891 2024-11-19T17:15:02,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/247d8aa7dfb147d4ae02dcea07c6f70b is 50, key is test_row_0/B:col10/1732036501738/Put/seqid=0 2024-11-19T17:15:02,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742185_1361 (size=12301) 2024-11-19T17:15:02,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T17:15:02,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:02,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:15:02,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:02,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:02,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:02,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:02,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T17:15:02,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T17:15:02,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:02,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:15:02,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:02,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:02,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:02,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:02,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:02,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036562403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:02,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:02,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:02,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:02,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:02,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036562413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:02,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:02,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036562414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:02,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T17:15:02,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:02,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:15:02,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:02,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:02,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:02,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:02,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/247d8aa7dfb147d4ae02dcea07c6f70b 2024-11-19T17:15:02,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/15d205dc6a6c47ada6e303638e1d2b4c is 50, key is test_row_0/C:col10/1732036501738/Put/seqid=0 2024-11-19T17:15:02,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742186_1362 (size=12301) 2024-11-19T17:15:02,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T17:15:02,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:02,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:15:02,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:02,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:02,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:02,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:02,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T17:15:02,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:02,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:15:02,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:02,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:02,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:02,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:02,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:02,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036562914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:02,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:02,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:02,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:02,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:02,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036562924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:02,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:02,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:02,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036562924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:02,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:02,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:02,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:02,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:02,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:03,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T17:15:03,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:03,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:15:03,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:03,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:03,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:03,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:03,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/15d205dc6a6c47ada6e303638e1d2b4c 2024-11-19T17:15:03,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/9dd98eab1cec4743aefded6dcd7b8891 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/9dd98eab1cec4743aefded6dcd7b8891 2024-11-19T17:15:03,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/9dd98eab1cec4743aefded6dcd7b8891, entries=150, sequenceid=526, filesize=12.0 K 2024-11-19T17:15:03,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/247d8aa7dfb147d4ae02dcea07c6f70b as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/247d8aa7dfb147d4ae02dcea07c6f70b 2024-11-19T17:15:03,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/247d8aa7dfb147d4ae02dcea07c6f70b, entries=150, sequenceid=526, filesize=12.0 K 2024-11-19T17:15:03,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/15d205dc6a6c47ada6e303638e1d2b4c as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/15d205dc6a6c47ada6e303638e1d2b4c 2024-11-19T17:15:03,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/15d205dc6a6c47ada6e303638e1d2b4c, entries=150, sequenceid=526, filesize=12.0 K 2024-11-19T17:15:03,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~318.89 KB/326544, currentSize=67.09 KB/68700 for c1ce7345860509b228081c672a073b0d in 1306ms, sequenceid=526, compaction requested=false 2024-11-19T17:15:03,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:15:03,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T17:15:03,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:03,162 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing c1ce7345860509b228081c672a073b0d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T17:15:03,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=A 2024-11-19T17:15:03,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:03,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=B 2024-11-19T17:15:03,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:03,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=C 2024-11-19T17:15:03,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:03,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/8100a4ef67c64cab848b0dbd55660438 is 50, key is test_row_0/A:col10/1732036501776/Put/seqid=0 2024-11-19T17:15:03,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742187_1363 (size=12301) 2024-11-19T17:15:03,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T17:15:03,572 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=542 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/8100a4ef67c64cab848b0dbd55660438 2024-11-19T17:15:03,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/7096717ba49f4862bf29748a8c2d6f0e is 50, key is test_row_0/B:col10/1732036501776/Put/seqid=0 2024-11-19T17:15:03,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742188_1364 (size=12301) 2024-11-19T17:15:03,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on c1ce7345860509b228081c672a073b0d 2024-11-19T17:15:03,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:15:03,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036563756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:03,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:03,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036563762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:03,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:03,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:03,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:03,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:03,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036563873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:03,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:03,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036563874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:03,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:03,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036563923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:03,928 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_2' in c1ce7345860509b228081c672a073b0d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:15:03,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:03,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:03,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:03,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:03,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036563933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:03,938 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_0' in c1ce7345860509b228081c672a073b0d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:15:03,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:03,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:03,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:03,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:03,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036563944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:03,947 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_0' in c1ce7345860509b228081c672a073b0d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:15:03,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:03,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:03,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:03,984 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=542 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/7096717ba49f4862bf29748a8c2d6f0e 2024-11-19T17:15:03,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/c8e941e1ef1b48ee86417e78e48e8c76 is 50, key is test_row_0/C:col10/1732036501776/Put/seqid=0 2024-11-19T17:15:04,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742189_1365 (size=12301) 2024-11-19T17:15:04,003 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=542 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/c8e941e1ef1b48ee86417e78e48e8c76 2024-11-19T17:15:04,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/8100a4ef67c64cab848b0dbd55660438 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/8100a4ef67c64cab848b0dbd55660438 2024-11-19T17:15:04,010 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/8100a4ef67c64cab848b0dbd55660438, entries=150, sequenceid=542, filesize=12.0 K 2024-11-19T17:15:04,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/7096717ba49f4862bf29748a8c2d6f0e as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/7096717ba49f4862bf29748a8c2d6f0e 2024-11-19T17:15:04,015 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/7096717ba49f4862bf29748a8c2d6f0e, entries=150, sequenceid=542, filesize=12.0 K 2024-11-19T17:15:04,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/c8e941e1ef1b48ee86417e78e48e8c76 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/c8e941e1ef1b48ee86417e78e48e8c76 2024-11-19T17:15:04,019 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/c8e941e1ef1b48ee86417e78e48e8c76, entries=150, sequenceid=542, filesize=12.0 K 2024-11-19T17:15:04,020 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c1ce7345860509b228081c672a073b0d in 858ms, sequenceid=542, compaction requested=true 2024-11-19T17:15:04,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:15:04,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:04,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-19T17:15:04,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-19T17:15:04,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-19T17:15:04,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8550 sec 2024-11-19T17:15:04,026 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 2.8600 sec 2024-11-19T17:15:04,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on c1ce7345860509b228081c672a073b0d 2024-11-19T17:15:04,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1ce7345860509b228081c672a073b0d 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-19T17:15:04,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=A 2024-11-19T17:15:04,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:04,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=B 2024-11-19T17:15:04,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:04,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=C 2024-11-19T17:15:04,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:04,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/536dceb998694936b22f1ce4569fd178 is 50, key is test_row_0/A:col10/1732036504086/Put/seqid=0 2024-11-19T17:15:04,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742190_1366 (size=12301) 2024-11-19T17:15:04,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:04,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036564116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:04,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:04,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:04,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:04,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:04,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036564120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:04,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:04,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:04,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:04,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:04,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036564233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:04,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:04,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:04,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:04,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036564233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:04,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:04,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:04,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036564444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:04,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:04,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:04,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:04,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036564444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:04,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:04,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:04,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:04,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:04,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/536dceb998694936b22f1ce4569fd178 2024-11-19T17:15:04,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/b25b0aa852894caeadbd871a3716ece6 is 50, key is test_row_0/B:col10/1732036504086/Put/seqid=0 2024-11-19T17:15:04,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742191_1367 (size=12301) 2024-11-19T17:15:04,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:04,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036564754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:04,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:04,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036564754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:04,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:04,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:04,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:04,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:04,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:04,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:04,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/b25b0aa852894caeadbd871a3716ece6 2024-11-19T17:15:04,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/dcd795898a074edc826256334786ee7b is 50, key is test_row_0/C:col10/1732036504086/Put/seqid=0 2024-11-19T17:15:04,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742192_1368 (size=12301) 2024-11-19T17:15:05,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:05,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036565263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:05,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:05,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036565264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:05,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:05,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:05,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:05,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:05,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:05,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:05,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T17:15:05,323 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:15:05,324 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:15:05,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees 2024-11-19T17:15:05,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-19T17:15:05,326 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:15:05,326 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:15:05,326 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:15:05,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/dcd795898a074edc826256334786ee7b 2024-11-19T17:15:05,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/536dceb998694936b22f1ce4569fd178 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/536dceb998694936b22f1ce4569fd178 2024-11-19T17:15:05,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/536dceb998694936b22f1ce4569fd178, entries=150, sequenceid=563, filesize=12.0 K 2024-11-19T17:15:05,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/b25b0aa852894caeadbd871a3716ece6 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/b25b0aa852894caeadbd871a3716ece6 2024-11-19T17:15:05,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/b25b0aa852894caeadbd871a3716ece6, entries=150, sequenceid=563, filesize=12.0 K 2024-11-19T17:15:05,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/dcd795898a074edc826256334786ee7b as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/dcd795898a074edc826256334786ee7b 2024-11-19T17:15:05,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/dcd795898a074edc826256334786ee7b, entries=150, sequenceid=563, filesize=12.0 K 2024-11-19T17:15:05,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for c1ce7345860509b228081c672a073b0d in 1257ms, sequenceid=563, compaction requested=true 2024-11-19T17:15:05,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:15:05,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1ce7345860509b228081c672a073b0d:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:15:05,345 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:15:05,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:05,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1ce7345860509b228081c672a073b0d:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:15:05,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:05,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1ce7345860509b228081c672a073b0d:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:15:05,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:05,345 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:15:05,346 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:15:05,346 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): c1ce7345860509b228081c672a073b0d/A is initiating minor compaction (all files) 2024-11-19T17:15:05,346 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1ce7345860509b228081c672a073b0d/A in TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:05,347 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:15:05,347 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/9f738e83a941483db04945354ec3a599, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/9dd98eab1cec4743aefded6dcd7b8891, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/8100a4ef67c64cab848b0dbd55660438, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/536dceb998694936b22f1ce4569fd178] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp, totalSize=48.6 K 2024-11-19T17:15:05,347 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): c1ce7345860509b228081c672a073b0d/B is initiating minor compaction (all files) 2024-11-19T17:15:05,347 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1ce7345860509b228081c672a073b0d/B in TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:05,347 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/225d817f2dfa47a39830d80ab39726b5, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/247d8aa7dfb147d4ae02dcea07c6f70b, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/7096717ba49f4862bf29748a8c2d6f0e, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/b25b0aa852894caeadbd871a3716ece6] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp, totalSize=48.6 K 2024-11-19T17:15:05,347 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f738e83a941483db04945354ec3a599, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732036500503 2024-11-19T17:15:05,347 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 225d817f2dfa47a39830d80ab39726b5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732036500503 2024-11-19T17:15:05,347 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9dd98eab1cec4743aefded6dcd7b8891, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1732036501726 2024-11-19T17:15:05,348 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 247d8aa7dfb147d4ae02dcea07c6f70b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1732036501726 2024-11-19T17:15:05,348 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8100a4ef67c64cab848b0dbd55660438, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=542, earliestPutTs=1732036501772 2024-11-19T17:15:05,348 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 7096717ba49f4862bf29748a8c2d6f0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=542, earliestPutTs=1732036501772 2024-11-19T17:15:05,348 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 536dceb998694936b22f1ce4569fd178, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1732036503753 2024-11-19T17:15:05,348 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting b25b0aa852894caeadbd871a3716ece6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1732036503753 2024-11-19T17:15:05,370 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1ce7345860509b228081c672a073b0d#B#compaction#316 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:05,370 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1ce7345860509b228081c672a073b0d#A#compaction#317 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:05,370 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/e1809ceac02540e1aa297f6c84fb7818 is 50, key is test_row_0/B:col10/1732036504086/Put/seqid=0 2024-11-19T17:15:05,370 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/52b05dd5258b4ef1a026f3ed565dd16a is 50, key is test_row_0/A:col10/1732036504086/Put/seqid=0 2024-11-19T17:15:05,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742194_1370 (size=13017) 2024-11-19T17:15:05,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742193_1369 (size=13017) 2024-11-19T17:15:05,389 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/52b05dd5258b4ef1a026f3ed565dd16a as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/52b05dd5258b4ef1a026f3ed565dd16a 2024-11-19T17:15:05,394 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1ce7345860509b228081c672a073b0d/A of c1ce7345860509b228081c672a073b0d into 52b05dd5258b4ef1a026f3ed565dd16a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:05,394 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:15:05,394 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., storeName=c1ce7345860509b228081c672a073b0d/A, priority=12, startTime=1732036505345; duration=0sec 2024-11-19T17:15:05,394 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:05,394 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1ce7345860509b228081c672a073b0d:A 2024-11-19T17:15:05,394 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:15:05,395 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:15:05,396 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): c1ce7345860509b228081c672a073b0d/C is initiating minor compaction (all files) 2024-11-19T17:15:05,396 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1ce7345860509b228081c672a073b0d/C in TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:05,396 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/b45bd95db0644b4bae11ad1732e91c91, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/15d205dc6a6c47ada6e303638e1d2b4c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/c8e941e1ef1b48ee86417e78e48e8c76, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/dcd795898a074edc826256334786ee7b] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp, totalSize=48.6 K 2024-11-19T17:15:05,396 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting b45bd95db0644b4bae11ad1732e91c91, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732036500503 2024-11-19T17:15:05,396 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15d205dc6a6c47ada6e303638e1d2b4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1732036501726 2024-11-19T17:15:05,397 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8e941e1ef1b48ee86417e78e48e8c76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=542, earliestPutTs=1732036501772 2024-11-19T17:15:05,397 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcd795898a074edc826256334786ee7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1732036503753 2024-11-19T17:15:05,405 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1ce7345860509b228081c672a073b0d#C#compaction#318 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:05,405 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/2a13b8374a7a4179aac22ff0637b5a51 is 50, key is test_row_0/C:col10/1732036504086/Put/seqid=0 2024-11-19T17:15:05,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742195_1371 (size=13017) 2024-11-19T17:15:05,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-19T17:15:05,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-19T17:15:05,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:05,479 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2902): Flushing c1ce7345860509b228081c672a073b0d 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T17:15:05,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=A 2024-11-19T17:15:05,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:05,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=B 2024-11-19T17:15:05,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:05,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=C 2024-11-19T17:15:05,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:05,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/2bf07bcf6a0b47eab3a2569b421e3410 is 50, key is test_row_0/A:col10/1732036504118/Put/seqid=0 2024-11-19T17:15:05,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742196_1372 (size=12301) 2024-11-19T17:15:05,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-19T17:15:05,789 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/e1809ceac02540e1aa297f6c84fb7818 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/e1809ceac02540e1aa297f6c84fb7818 2024-11-19T17:15:05,793 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1ce7345860509b228081c672a073b0d/B of c1ce7345860509b228081c672a073b0d into e1809ceac02540e1aa297f6c84fb7818(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:05,793 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:15:05,793 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., storeName=c1ce7345860509b228081c672a073b0d/B, priority=12, startTime=1732036505345; duration=0sec 2024-11-19T17:15:05,793 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:05,793 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1ce7345860509b228081c672a073b0d:B 2024-11-19T17:15:05,814 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/2a13b8374a7a4179aac22ff0637b5a51 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/2a13b8374a7a4179aac22ff0637b5a51 2024-11-19T17:15:05,818 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1ce7345860509b228081c672a073b0d/C of c1ce7345860509b228081c672a073b0d into 2a13b8374a7a4179aac22ff0637b5a51(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:05,818 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:15:05,818 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., storeName=c1ce7345860509b228081c672a073b0d/C, priority=12, startTime=1732036505345; duration=0sec 2024-11-19T17:15:05,818 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:05,818 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1ce7345860509b228081c672a073b0d:C 2024-11-19T17:15:05,888 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/2bf07bcf6a0b47eab3a2569b421e3410 2024-11-19T17:15:05,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/3e9e678a1d4a443d86e9ead3eeacb7b6 is 50, key is test_row_0/B:col10/1732036504118/Put/seqid=0 2024-11-19T17:15:05,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742197_1373 (size=12301) 2024-11-19T17:15:05,900 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/3e9e678a1d4a443d86e9ead3eeacb7b6 2024-11-19T17:15:05,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/d44a8f5f470547c69252713f89a2b558 is 50, key is test_row_0/C:col10/1732036504118/Put/seqid=0 2024-11-19T17:15:05,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742198_1374 (size=12301) 2024-11-19T17:15:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on c1ce7345860509b228081c672a073b0d 2024-11-19T17:15:05,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:15:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-19T17:15:05,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036565983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:05,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036565984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:05,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036505967, max timestamp=1732036505988Num uniques 859; 2024-11-19T17:15:05,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036505967, max timestamp=1732036505988Num uniques 859; 2024-11-19T17:15:05,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:15:05,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:15:05,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036505967, max timestamp=1732036505988Num uniques 859; 2024-11-19T17:15:05,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:15:05,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:05,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:05,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:06,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036566068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:06,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:06,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:06,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:06,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036566093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:06,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:06,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036566093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:06,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:06,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:06,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:06,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:06,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:06,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036566174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:06,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:06,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:06,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:06,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1732036566274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:06,277 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_1' in c1ce7345860509b228081c672a073b0d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:15:06,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:06,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:06,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:06,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60474 deadline: 1732036566283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:06,286 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_1' in c1ce7345860509b228081c672a073b0d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:15:06,286 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:06,286 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:06,286 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:06,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036566303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:06,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036566303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:06,315 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/d44a8f5f470547c69252713f89a2b558 2024-11-19T17:15:06,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/2bf07bcf6a0b47eab3a2569b421e3410 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/2bf07bcf6a0b47eab3a2569b421e3410 2024-11-19T17:15:06,323 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/2bf07bcf6a0b47eab3a2569b421e3410, entries=150, sequenceid=575, filesize=12.0 K 2024-11-19T17:15:06,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/3e9e678a1d4a443d86e9ead3eeacb7b6 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/3e9e678a1d4a443d86e9ead3eeacb7b6 2024-11-19T17:15:06,328 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/3e9e678a1d4a443d86e9ead3eeacb7b6, entries=150, sequenceid=575, filesize=12.0 K 2024-11-19T17:15:06,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/d44a8f5f470547c69252713f89a2b558 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/d44a8f5f470547c69252713f89a2b558 2024-11-19T17:15:06,334 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/d44a8f5f470547c69252713f89a2b558, entries=150, sequenceid=575, filesize=12.0 K 2024-11-19T17:15:06,335 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for c1ce7345860509b228081c672a073b0d in 856ms, sequenceid=575, compaction requested=false 2024-11-19T17:15:06,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2603): Flush status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:15:06,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:06,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=89 2024-11-19T17:15:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=89 2024-11-19T17:15:06,338 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-19T17:15:06,338 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0100 sec 2024-11-19T17:15:06,341 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees in 1.0150 sec 2024-11-19T17:15:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on c1ce7345860509b228081c672a073b0d 2024-11-19T17:15:06,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1ce7345860509b228081c672a073b0d 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-19T17:15:06,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=A 2024-11-19T17:15:06,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:06,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=B 2024-11-19T17:15:06,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:06,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=C 2024-11-19T17:15:06,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:06,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/6df1d089bb514b268995e40bbdcdf4ca is 50, key is test_row_0/A:col10/1732036506001/Put/seqid=0 2024-11-19T17:15:06,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036566417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:06,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:06,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:06,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:06,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742199_1375 (size=12301) 2024-11-19T17:15:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-19T17:15:06,464 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:15:06,465 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:15:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees 2024-11-19T17:15:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-19T17:15:06,467 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:15:06,468 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:15:06,468 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:15:06,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:06,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036566534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:06,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:06,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:06,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:06,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-19T17:15:06,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036566614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:06,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:06,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036566614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:06,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:06,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:06,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:06,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:06,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:06,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-19T17:15:06,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:06,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:15:06,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:06,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:06,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:06,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036566744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:06,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:06,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-19T17:15:06,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:06,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:15:06,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:06,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:06,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:06,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:06,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-19T17:15:06,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=614 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/6df1d089bb514b268995e40bbdcdf4ca 2024-11-19T17:15:06,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/2f491f0277d845969a003723defe3e07 is 50, key is test_row_0/B:col10/1732036506001/Put/seqid=0 2024-11-19T17:15:06,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742200_1376 (size=12301) 2024-11-19T17:15:06,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-19T17:15:06,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:06,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:15:06,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:06,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:06,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:07,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036567054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:07,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:07,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:07,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:07,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-19T17:15:07,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:07,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:15:07,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:07,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:07,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:07,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-19T17:15:07,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60510 deadline: 1732036567123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:07,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:07,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1732036567123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:07,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:07,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:07,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:07,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:07,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:07,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-19T17:15:07,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:07,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:15:07,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:07,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:07,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:07,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=614 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/2f491f0277d845969a003723defe3e07 2024-11-19T17:15:07,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/52053f39914a464badb9673089fc9ffc is 50, key is test_row_0/C:col10/1732036506001/Put/seqid=0 2024-11-19T17:15:07,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742201_1377 (size=12301) 2024-11-19T17:15:07,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-19T17:15:07,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:07,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:15:07,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:07,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:07,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:07,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:07,391 INFO [Thread-1390 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1390. 2024-11-19T17:15:07,391 DEBUG [Thread-1390 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:15:07,391 DEBUG [Thread-1390 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:07,391 DEBUG [Thread-1390 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:07,391 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:15:07,393 INFO [Thread-1392 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1392. 2024-11-19T17:15:07,393 DEBUG [Thread-1392 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:15:07,393 INFO [Thread-1396 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1396. 2024-11-19T17:15:07,394 DEBUG [Thread-1392 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:07,394 DEBUG [Thread-1396 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:15:07,394 DEBUG [Thread-1396 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:07,394 DEBUG [Thread-1392 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:07,394 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:15:07,394 DEBUG [Thread-1396 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:07,394 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:15:07,395 INFO [Thread-1398 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1398. 2024-11-19T17:15:07,396 DEBUG [Thread-1398 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:15:07,396 DEBUG [Thread-1398 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:07,396 DEBUG [Thread-1398 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:07,396 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:15:07,397 INFO [Thread-1394 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1394. 2024-11-19T17:15:07,397 DEBUG [Thread-1394 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:15:07,397 DEBUG [Thread-1394 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:07,397 DEBUG [Thread-1394 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:07,397 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:15:07,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-19T17:15:07,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:07,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:15:07,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:07,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:07,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:07,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:07,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:07,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1732036567563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:07,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:07,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:07,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:07,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-19T17:15:07,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=614 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/52053f39914a464badb9673089fc9ffc 2024-11-19T17:15:07,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/6df1d089bb514b268995e40bbdcdf4ca as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/6df1d089bb514b268995e40bbdcdf4ca 2024-11-19T17:15:07,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/6df1d089bb514b268995e40bbdcdf4ca, entries=150, sequenceid=614, filesize=12.0 K 2024-11-19T17:15:07,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/2f491f0277d845969a003723defe3e07 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/2f491f0277d845969a003723defe3e07 2024-11-19T17:15:07,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/2f491f0277d845969a003723defe3e07, entries=150, sequenceid=614, filesize=12.0 K 2024-11-19T17:15:07,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/52053f39914a464badb9673089fc9ffc as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/52053f39914a464badb9673089fc9ffc 2024-11-19T17:15:07,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/52053f39914a464badb9673089fc9ffc, entries=150, sequenceid=614, filesize=12.0 K 2024-11-19T17:15:07,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for c1ce7345860509b228081c672a073b0d in 1285ms, sequenceid=614, compaction requested=true 2024-11-19T17:15:07,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:15:07,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1ce7345860509b228081c672a073b0d:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:15:07,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:07,671 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:07,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1ce7345860509b228081c672a073b0d:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:15:07,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:07,672 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:07,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1ce7345860509b228081c672a073b0d:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:15:07,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:07,672 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:07,672 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:07,672 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): c1ce7345860509b228081c672a073b0d/B is initiating minor compaction (all files) 2024-11-19T17:15:07,672 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): c1ce7345860509b228081c672a073b0d/A is initiating minor compaction (all files) 2024-11-19T17:15:07,672 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1ce7345860509b228081c672a073b0d/B in TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:07,672 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1ce7345860509b228081c672a073b0d/A in TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:07,672 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/e1809ceac02540e1aa297f6c84fb7818, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/3e9e678a1d4a443d86e9ead3eeacb7b6, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/2f491f0277d845969a003723defe3e07] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp, totalSize=36.7 K 2024-11-19T17:15:07,672 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/52b05dd5258b4ef1a026f3ed565dd16a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/2bf07bcf6a0b47eab3a2569b421e3410, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/6df1d089bb514b268995e40bbdcdf4ca] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp, totalSize=36.7 K 2024-11-19T17:15:07,673 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52b05dd5258b4ef1a026f3ed565dd16a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1732036503753 2024-11-19T17:15:07,673 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting e1809ceac02540e1aa297f6c84fb7818, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1732036503753 2024-11-19T17:15:07,673 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bf07bcf6a0b47eab3a2569b421e3410, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=575, earliestPutTs=1732036504112 2024-11-19T17:15:07,673 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e9e678a1d4a443d86e9ead3eeacb7b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=575, earliestPutTs=1732036504112 2024-11-19T17:15:07,673 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6df1d089bb514b268995e40bbdcdf4ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=614, earliestPutTs=1732036506001 2024-11-19T17:15:07,673 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f491f0277d845969a003723defe3e07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=614, earliestPutTs=1732036506001 2024-11-19T17:15:07,679 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1ce7345860509b228081c672a073b0d#B#compaction#325 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:07,679 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1ce7345860509b228081c672a073b0d#A#compaction#326 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:07,679 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/47c4c0736fbc46c0ba48261cfe07d94d is 50, key is test_row_0/B:col10/1732036506001/Put/seqid=0 2024-11-19T17:15:07,679 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/e9d1afe3ae98409190b3378d92d0fe9e is 50, key is test_row_0/A:col10/1732036506001/Put/seqid=0 2024-11-19T17:15:07,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742202_1378 (size=13119) 2024-11-19T17:15:07,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742203_1379 (size=13119) 2024-11-19T17:15:07,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-19T17:15:07,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:07,693 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2902): Flushing c1ce7345860509b228081c672a073b0d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T17:15:07,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=A 2024-11-19T17:15:07,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:07,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=B 2024-11-19T17:15:07,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:07,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=C 2024-11-19T17:15:07,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:07,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/8dbc50fa2e07499599ddfdf0beb38779 is 50, key is test_row_0/A:col10/1732036506415/Put/seqid=0 2024-11-19T17:15:07,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742204_1380 (size=9857) 2024-11-19T17:15:08,090 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/47c4c0736fbc46c0ba48261cfe07d94d as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/47c4c0736fbc46c0ba48261cfe07d94d 2024-11-19T17:15:08,090 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/e9d1afe3ae98409190b3378d92d0fe9e as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/e9d1afe3ae98409190b3378d92d0fe9e 2024-11-19T17:15:08,094 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1ce7345860509b228081c672a073b0d/A of c1ce7345860509b228081c672a073b0d into e9d1afe3ae98409190b3378d92d0fe9e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:08,094 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1ce7345860509b228081c672a073b0d/B of c1ce7345860509b228081c672a073b0d into 47c4c0736fbc46c0ba48261cfe07d94d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:08,094 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:15:08,094 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:15:08,094 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., storeName=c1ce7345860509b228081c672a073b0d/B, priority=13, startTime=1732036507671; duration=0sec 2024-11-19T17:15:08,094 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., storeName=c1ce7345860509b228081c672a073b0d/A, priority=13, startTime=1732036507671; duration=0sec 2024-11-19T17:15:08,094 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:08,094 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:08,094 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1ce7345860509b228081c672a073b0d:B 2024-11-19T17:15:08,094 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1ce7345860509b228081c672a073b0d:A 2024-11-19T17:15:08,094 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:08,095 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:08,095 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): c1ce7345860509b228081c672a073b0d/C is initiating minor compaction (all files) 2024-11-19T17:15:08,095 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1ce7345860509b228081c672a073b0d/C in TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:08,095 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/2a13b8374a7a4179aac22ff0637b5a51, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/d44a8f5f470547c69252713f89a2b558, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/52053f39914a464badb9673089fc9ffc] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp, totalSize=36.7 K 2024-11-19T17:15:08,095 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a13b8374a7a4179aac22ff0637b5a51, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1732036503753 2024-11-19T17:15:08,096 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting d44a8f5f470547c69252713f89a2b558, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=575, earliestPutTs=1732036504112 2024-11-19T17:15:08,096 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 52053f39914a464badb9673089fc9ffc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=614, earliestPutTs=1732036506001 2024-11-19T17:15:08,100 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=624 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/8dbc50fa2e07499599ddfdf0beb38779 2024-11-19T17:15:08,102 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1ce7345860509b228081c672a073b0d#C#compaction#328 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:08,102 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/b53746612fd742189767fc17592c37a4 is 50, key is test_row_0/C:col10/1732036506001/Put/seqid=0 2024-11-19T17:15:08,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/d9431356c46e4c8da5bbcfbed2696458 is 50, key is test_row_0/B:col10/1732036506415/Put/seqid=0 2024-11-19T17:15:08,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742205_1381 (size=13119) 2024-11-19T17:15:08,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742206_1382 (size=9857) 2024-11-19T17:15:08,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on c1ce7345860509b228081c672a073b0d 2024-11-19T17:15:08,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. as already flushing 2024-11-19T17:15:08,135 INFO [Thread-1379 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1379. 2024-11-19T17:15:08,135 INFO [Thread-1381 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1381. 2024-11-19T17:15:08,135 DEBUG [Thread-1379 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:15:08,135 DEBUG [Thread-1381 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:15:08,135 DEBUG [Thread-1379 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:08,135 DEBUG [Thread-1381 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:08,135 DEBUG [Thread-1381 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:08,135 DEBUG [Thread-1379 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:08,135 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:15:08,135 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:15:08,296 INFO [Thread-1385 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1385. 2024-11-19T17:15:08,296 DEBUG [Thread-1385 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:15:08,296 DEBUG [Thread-1385 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:08,296 DEBUG [Thread-1385 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:08,296 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:15:08,315 INFO [Thread-1383 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1383. 2024-11-19T17:15:08,315 DEBUG [Thread-1383 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:15:08,315 DEBUG [Thread-1383 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:08,315 DEBUG [Thread-1383 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:08,315 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:15:08,510 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=624 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/d9431356c46e4c8da5bbcfbed2696458 2024-11-19T17:15:08,510 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/b53746612fd742189767fc17592c37a4 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/b53746612fd742189767fc17592c37a4 2024-11-19T17:15:08,515 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1ce7345860509b228081c672a073b0d/C of c1ce7345860509b228081c672a073b0d into b53746612fd742189767fc17592c37a4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:08,515 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:15:08,515 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., storeName=c1ce7345860509b228081c672a073b0d/C, priority=13, startTime=1732036507672; duration=0sec 2024-11-19T17:15:08,515 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:08,515 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1ce7345860509b228081c672a073b0d:C 2024-11-19T17:15:08,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/e45e8103f6b945a3afea590787166350 is 50, key is test_row_0/C:col10/1732036506415/Put/seqid=0 2024-11-19T17:15:08,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742207_1383 (size=9857) 2024-11-19T17:15:08,576 INFO [Thread-1387 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1387. 2024-11-19T17:15:08,576 DEBUG [Thread-1387 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:15:08,576 DEBUG [Thread-1387 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:08,576 DEBUG [Thread-1387 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:08,576 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:15:08,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-19T17:15:08,927 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=624 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/e45e8103f6b945a3afea590787166350 2024-11-19T17:15:08,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/8dbc50fa2e07499599ddfdf0beb38779 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/8dbc50fa2e07499599ddfdf0beb38779 2024-11-19T17:15:08,934 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/8dbc50fa2e07499599ddfdf0beb38779, entries=100, sequenceid=624, filesize=9.6 K 2024-11-19T17:15:08,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/d9431356c46e4c8da5bbcfbed2696458 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/d9431356c46e4c8da5bbcfbed2696458 2024-11-19T17:15:08,938 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/d9431356c46e4c8da5bbcfbed2696458, entries=100, sequenceid=624, filesize=9.6 K 2024-11-19T17:15:08,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/e45e8103f6b945a3afea590787166350 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/e45e8103f6b945a3afea590787166350 2024-11-19T17:15:08,942 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/e45e8103f6b945a3afea590787166350, entries=100, sequenceid=624, filesize=9.6 K 2024-11-19T17:15:08,943 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=33.54 KB/34350 for c1ce7345860509b228081c672a073b0d in 1250ms, sequenceid=624, compaction requested=false 2024-11-19T17:15:08,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2603): Flush status journal for c1ce7345860509b228081c672a073b0d: 2024-11-19T17:15:08,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:08,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=91 2024-11-19T17:15:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=91 2024-11-19T17:15:08,946 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-19T17:15:08,946 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4760 sec 2024-11-19T17:15:08,947 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees in 2.4810 sec 2024-11-19T17:15:10,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-19T17:15:10,634 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:15:10,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T17:15:10,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 148 2024-11-19T17:15:10,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 134 2024-11-19T17:15:10,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-19T17:15:10,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-19T17:15:10,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 177 2024-11-19T17:15:10,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T17:15:10,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2975 2024-11-19T17:15:10,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2952 2024-11-19T17:15:10,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2996 2024-11-19T17:15:10,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2975 2024-11-19T17:15:10,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2968 2024-11-19T17:15:10,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T17:15:10,635 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T17:15:10,635 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T17:15:10,635 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:10,635 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:10,635 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:15:10,636 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T17:15:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T17:15:10,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-19T17:15:10,639 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732036510639"}]},"ts":"1732036510639"} 2024-11-19T17:15:10,640 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T17:15:10,640 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T17:15:10,641 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T17:15:10,642 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1ce7345860509b228081c672a073b0d, UNASSIGN}] 2024-11-19T17:15:10,642 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1ce7345860509b228081c672a073b0d, UNASSIGN 2024-11-19T17:15:10,643 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=c1ce7345860509b228081c672a073b0d, regionState=CLOSING, regionLocation=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:10,644 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1ce7345860509b228081c672a073b0d, UNASSIGN because future has completed 2024-11-19T17:15:10,644 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T17:15:10,645 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473}] 2024-11-19T17:15:10,709 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T17:15:10,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-19T17:15:10,797 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close c1ce7345860509b228081c672a073b0d 2024-11-19T17:15:10,797 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T17:15:10,797 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing c1ce7345860509b228081c672a073b0d, disabling compactions & flushes 2024-11-19T17:15:10,797 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:10,797 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:10,797 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. after waiting 0 ms 2024-11-19T17:15:10,797 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:10,797 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(2902): Flushing c1ce7345860509b228081c672a073b0d 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T17:15:10,797 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=A 2024-11-19T17:15:10,798 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:10,798 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=B 2024-11-19T17:15:10,798 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:10,798 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1ce7345860509b228081c672a073b0d, store=C 2024-11-19T17:15:10,798 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:10,801 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/e205ac54f2634e5dbfe19d2bb3e3013d is 50, key is test_row_1/A:col10/1732036508314/Put/seqid=0 2024-11-19T17:15:10,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742208_1384 (size=9857) 2024-11-19T17:15:10,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-19T17:15:11,205 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/e205ac54f2634e5dbfe19d2bb3e3013d 2024-11-19T17:15:11,211 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/eb7939cd96314a02a7c528a80ebe15fc is 50, key is test_row_1/B:col10/1732036508314/Put/seqid=0 2024-11-19T17:15:11,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742209_1385 (size=9857) 2024-11-19T17:15:11,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-19T17:15:11,615 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/eb7939cd96314a02a7c528a80ebe15fc 2024-11-19T17:15:11,621 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/7392662e4e014ca9a262166449d575af is 50, key is test_row_1/C:col10/1732036508314/Put/seqid=0 2024-11-19T17:15:11,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742210_1386 (size=9857) 2024-11-19T17:15:11,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-19T17:15:12,025 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/7392662e4e014ca9a262166449d575af 2024-11-19T17:15:12,029 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/A/e205ac54f2634e5dbfe19d2bb3e3013d as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/e205ac54f2634e5dbfe19d2bb3e3013d 2024-11-19T17:15:12,032 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/e205ac54f2634e5dbfe19d2bb3e3013d, entries=100, sequenceid=635, filesize=9.6 K 2024-11-19T17:15:12,033 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/B/eb7939cd96314a02a7c528a80ebe15fc as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/eb7939cd96314a02a7c528a80ebe15fc 2024-11-19T17:15:12,035 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/eb7939cd96314a02a7c528a80ebe15fc, entries=100, sequenceid=635, filesize=9.6 K 2024-11-19T17:15:12,036 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/.tmp/C/7392662e4e014ca9a262166449d575af as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/7392662e4e014ca9a262166449d575af 2024-11-19T17:15:12,038 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/7392662e4e014ca9a262166449d575af, entries=100, sequenceid=635, filesize=9.6 K 2024-11-19T17:15:12,039 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c1ce7345860509b228081c672a073b0d in 1242ms, sequenceid=635, compaction requested=true 2024-11-19T17:15:12,039 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/f6bead26a6eb41f393736023ae51c14b, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/7597280727a84e2ea8cee5784dbeaae5, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/4b665a00db49462ba39db5d309285067, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/fdc1cddfed71443dad8089e46ea656d7, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/b2754535260f4b83a38436038340b076, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/119d0817e1ac43c48e8a2f4e37278fa7, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/2d98932586504a0c8ebe21b513666bd7, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/6178faa8886a4e85b7f412577cdb426d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/1ff8553674c14a0c8789f77c41f4cd4e, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/a635c9962c344bb78da80daca4d41631, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/12d540fdb7b344448458f5572423795d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/da551f0793eb4cabae3c9981dd680b66, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/c767fecdc05b4f22a30f7abf86c03dee, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/4fc232dc5001497dbc9c36f309d33d17, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/0234dd82a57d41419c24b048a2bebc27, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/c7364cb6539b4ee98c47a8e6ab064f1a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/e86e292eaeca40bf954202053e01b78e, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/9f738e83a941483db04945354ec3a599, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/9dd98eab1cec4743aefded6dcd7b8891, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/8100a4ef67c64cab848b0dbd55660438, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/52b05dd5258b4ef1a026f3ed565dd16a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/536dceb998694936b22f1ce4569fd178, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/2bf07bcf6a0b47eab3a2569b421e3410, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/6df1d089bb514b268995e40bbdcdf4ca] to archive 2024-11-19T17:15:12,040 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:15:12,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/f6bead26a6eb41f393736023ae51c14b to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/f6bead26a6eb41f393736023ae51c14b 2024-11-19T17:15:12,042 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/7597280727a84e2ea8cee5784dbeaae5 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/7597280727a84e2ea8cee5784dbeaae5 2024-11-19T17:15:12,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/4b665a00db49462ba39db5d309285067 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/4b665a00db49462ba39db5d309285067 2024-11-19T17:15:12,044 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/fdc1cddfed71443dad8089e46ea656d7 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/fdc1cddfed71443dad8089e46ea656d7 2024-11-19T17:15:12,044 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/b2754535260f4b83a38436038340b076 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/b2754535260f4b83a38436038340b076 2024-11-19T17:15:12,045 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/119d0817e1ac43c48e8a2f4e37278fa7 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/119d0817e1ac43c48e8a2f4e37278fa7 2024-11-19T17:15:12,046 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/2d98932586504a0c8ebe21b513666bd7 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/2d98932586504a0c8ebe21b513666bd7 2024-11-19T17:15:12,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/6178faa8886a4e85b7f412577cdb426d to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/6178faa8886a4e85b7f412577cdb426d 2024-11-19T17:15:12,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/1ff8553674c14a0c8789f77c41f4cd4e to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/1ff8553674c14a0c8789f77c41f4cd4e 2024-11-19T17:15:12,048 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/a635c9962c344bb78da80daca4d41631 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/a635c9962c344bb78da80daca4d41631 2024-11-19T17:15:12,049 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/12d540fdb7b344448458f5572423795d to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/12d540fdb7b344448458f5572423795d 2024-11-19T17:15:12,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/da551f0793eb4cabae3c9981dd680b66 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/da551f0793eb4cabae3c9981dd680b66 2024-11-19T17:15:12,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/c767fecdc05b4f22a30f7abf86c03dee to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/c767fecdc05b4f22a30f7abf86c03dee 2024-11-19T17:15:12,051 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/4fc232dc5001497dbc9c36f309d33d17 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/4fc232dc5001497dbc9c36f309d33d17 2024-11-19T17:15:12,052 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/0234dd82a57d41419c24b048a2bebc27 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/0234dd82a57d41419c24b048a2bebc27 2024-11-19T17:15:12,053 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/c7364cb6539b4ee98c47a8e6ab064f1a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/c7364cb6539b4ee98c47a8e6ab064f1a 2024-11-19T17:15:12,054 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/e86e292eaeca40bf954202053e01b78e to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/e86e292eaeca40bf954202053e01b78e 2024-11-19T17:15:12,054 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/9f738e83a941483db04945354ec3a599 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/9f738e83a941483db04945354ec3a599 2024-11-19T17:15:12,055 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/9dd98eab1cec4743aefded6dcd7b8891 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/9dd98eab1cec4743aefded6dcd7b8891 2024-11-19T17:15:12,056 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/8100a4ef67c64cab848b0dbd55660438 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/8100a4ef67c64cab848b0dbd55660438 2024-11-19T17:15:12,056 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/52b05dd5258b4ef1a026f3ed565dd16a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/52b05dd5258b4ef1a026f3ed565dd16a 2024-11-19T17:15:12,057 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/536dceb998694936b22f1ce4569fd178 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/536dceb998694936b22f1ce4569fd178 2024-11-19T17:15:12,058 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/2bf07bcf6a0b47eab3a2569b421e3410 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/2bf07bcf6a0b47eab3a2569b421e3410 2024-11-19T17:15:12,059 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/6df1d089bb514b268995e40bbdcdf4ca to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/6df1d089bb514b268995e40bbdcdf4ca 2024-11-19T17:15:12,060 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/ed15f3d24a5c4472914fd4a3ddfa65b1, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/c57894828ce74820a443cb923d02e053, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/145ead5330ee4f23a4426aa03571580a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/9eef32356aa1469bb0e5bf7ed405f59e, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/f50d59af2fa74de4ae5ae65d59bec3ce, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/1e9fec8ad3dc42e187fb02a2af8fa14c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/e575b39b5e6f43569445a8fb9bc1a933, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/bed3915a70de42f086edb584d7ba4f27, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/b885e635aabf48ea92e5c82bb89bdaad, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/07cedf54c63d4f0688ec3db1fc8f95a9, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/cf8ec834b57d40c2b93a2259233d5ea9, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/60bc273fe368443db990ce87e538f063, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/7c3b14ddaade419eb3d3b614d900ec48, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/14e8700997ce45d3a2823bcaaccefb40, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/9c12e6dcdc6d4dbeb2f1d8d396b794cc, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/eec6d8189e0440eba1e179404e397bbd, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/225d817f2dfa47a39830d80ab39726b5, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/3395f0bc41ec4d86b9d8a61423a02d6d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/247d8aa7dfb147d4ae02dcea07c6f70b, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/7096717ba49f4862bf29748a8c2d6f0e, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/e1809ceac02540e1aa297f6c84fb7818, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/b25b0aa852894caeadbd871a3716ece6, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/3e9e678a1d4a443d86e9ead3eeacb7b6, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/2f491f0277d845969a003723defe3e07] to archive 2024-11-19T17:15:12,061 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:15:12,062 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/ed15f3d24a5c4472914fd4a3ddfa65b1 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/ed15f3d24a5c4472914fd4a3ddfa65b1 2024-11-19T17:15:12,062 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/c57894828ce74820a443cb923d02e053 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/c57894828ce74820a443cb923d02e053 2024-11-19T17:15:12,063 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/145ead5330ee4f23a4426aa03571580a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/145ead5330ee4f23a4426aa03571580a 2024-11-19T17:15:12,064 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/9eef32356aa1469bb0e5bf7ed405f59e to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/9eef32356aa1469bb0e5bf7ed405f59e 2024-11-19T17:15:12,065 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/f50d59af2fa74de4ae5ae65d59bec3ce to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/f50d59af2fa74de4ae5ae65d59bec3ce 2024-11-19T17:15:12,065 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/1e9fec8ad3dc42e187fb02a2af8fa14c to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/1e9fec8ad3dc42e187fb02a2af8fa14c 2024-11-19T17:15:12,066 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/e575b39b5e6f43569445a8fb9bc1a933 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/e575b39b5e6f43569445a8fb9bc1a933 2024-11-19T17:15:12,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/bed3915a70de42f086edb584d7ba4f27 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/bed3915a70de42f086edb584d7ba4f27 2024-11-19T17:15:12,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/b885e635aabf48ea92e5c82bb89bdaad to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/b885e635aabf48ea92e5c82bb89bdaad 2024-11-19T17:15:12,068 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/07cedf54c63d4f0688ec3db1fc8f95a9 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/07cedf54c63d4f0688ec3db1fc8f95a9 2024-11-19T17:15:12,070 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/cf8ec834b57d40c2b93a2259233d5ea9 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/cf8ec834b57d40c2b93a2259233d5ea9 2024-11-19T17:15:12,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/60bc273fe368443db990ce87e538f063 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/60bc273fe368443db990ce87e538f063 2024-11-19T17:15:12,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/7c3b14ddaade419eb3d3b614d900ec48 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/7c3b14ddaade419eb3d3b614d900ec48 2024-11-19T17:15:12,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/14e8700997ce45d3a2823bcaaccefb40 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/14e8700997ce45d3a2823bcaaccefb40 2024-11-19T17:15:12,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/9c12e6dcdc6d4dbeb2f1d8d396b794cc to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/9c12e6dcdc6d4dbeb2f1d8d396b794cc 2024-11-19T17:15:12,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/eec6d8189e0440eba1e179404e397bbd to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/eec6d8189e0440eba1e179404e397bbd 2024-11-19T17:15:12,075 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/225d817f2dfa47a39830d80ab39726b5 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/225d817f2dfa47a39830d80ab39726b5 2024-11-19T17:15:12,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/3395f0bc41ec4d86b9d8a61423a02d6d to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/3395f0bc41ec4d86b9d8a61423a02d6d 2024-11-19T17:15:12,077 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/247d8aa7dfb147d4ae02dcea07c6f70b to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/247d8aa7dfb147d4ae02dcea07c6f70b 2024-11-19T17:15:12,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/7096717ba49f4862bf29748a8c2d6f0e to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/7096717ba49f4862bf29748a8c2d6f0e 2024-11-19T17:15:12,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/e1809ceac02540e1aa297f6c84fb7818 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/e1809ceac02540e1aa297f6c84fb7818 2024-11-19T17:15:12,080 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/b25b0aa852894caeadbd871a3716ece6 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/b25b0aa852894caeadbd871a3716ece6 2024-11-19T17:15:12,080 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/3e9e678a1d4a443d86e9ead3eeacb7b6 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/3e9e678a1d4a443d86e9ead3eeacb7b6 2024-11-19T17:15:12,081 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/2f491f0277d845969a003723defe3e07 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/2f491f0277d845969a003723defe3e07 2024-11-19T17:15:12,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/faf4c31fffe04a1897fc1f390b1314ee, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/2c966e10909447ecbc8c241ad88517df, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/49d9d1f76f664682a238f4cc502ca439, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/77f7463d58f34856a49da6efb672db16, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/9c398c079d0247a3842a6462d14d8b4c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/0d1da9f998c74dcbae934a1d5bd5584b, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/75f7b6328e9e422692e0754acc5470a6, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/f47db503b7674b2d8e95aa1ecdcc05d4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/50693a1c58b64cb79ecf4c79cdb95a48, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/4758be2ed4ab4e109f79b7be0d695f1f, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/afef44b4499a471e99dcf2126230e470, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/50d9012ef91c498f97b681156328718c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/1d7dbb9fd8bb442e92c61e9ea3c775ca, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/1430bc0cec6c4025b812c843d05f0b09, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/be83d1b679b746e28639269040d78430, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/8f03d7f0711d43bab4168b9701ed6a26, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/b45bd95db0644b4bae11ad1732e91c91, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/3ba9c918259046a7be5faeda129904e7, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/15d205dc6a6c47ada6e303638e1d2b4c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/c8e941e1ef1b48ee86417e78e48e8c76, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/2a13b8374a7a4179aac22ff0637b5a51, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/dcd795898a074edc826256334786ee7b, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/d44a8f5f470547c69252713f89a2b558, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/52053f39914a464badb9673089fc9ffc] to archive 2024-11-19T17:15:12,084 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:15:12,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/faf4c31fffe04a1897fc1f390b1314ee to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/faf4c31fffe04a1897fc1f390b1314ee 2024-11-19T17:15:12,086 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/2c966e10909447ecbc8c241ad88517df to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/2c966e10909447ecbc8c241ad88517df 2024-11-19T17:15:12,087 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/49d9d1f76f664682a238f4cc502ca439 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/49d9d1f76f664682a238f4cc502ca439 2024-11-19T17:15:12,087 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/77f7463d58f34856a49da6efb672db16 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/77f7463d58f34856a49da6efb672db16 2024-11-19T17:15:12,088 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/9c398c079d0247a3842a6462d14d8b4c to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/9c398c079d0247a3842a6462d14d8b4c 2024-11-19T17:15:12,089 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/0d1da9f998c74dcbae934a1d5bd5584b to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/0d1da9f998c74dcbae934a1d5bd5584b 2024-11-19T17:15:12,090 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/75f7b6328e9e422692e0754acc5470a6 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/75f7b6328e9e422692e0754acc5470a6 2024-11-19T17:15:12,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/f47db503b7674b2d8e95aa1ecdcc05d4 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/f47db503b7674b2d8e95aa1ecdcc05d4 2024-11-19T17:15:12,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/50693a1c58b64cb79ecf4c79cdb95a48 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/50693a1c58b64cb79ecf4c79cdb95a48 2024-11-19T17:15:12,092 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/4758be2ed4ab4e109f79b7be0d695f1f to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/4758be2ed4ab4e109f79b7be0d695f1f 2024-11-19T17:15:12,093 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/afef44b4499a471e99dcf2126230e470 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/afef44b4499a471e99dcf2126230e470 2024-11-19T17:15:12,094 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/50d9012ef91c498f97b681156328718c to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/50d9012ef91c498f97b681156328718c 2024-11-19T17:15:12,095 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/1d7dbb9fd8bb442e92c61e9ea3c775ca to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/1d7dbb9fd8bb442e92c61e9ea3c775ca 2024-11-19T17:15:12,095 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/1430bc0cec6c4025b812c843d05f0b09 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/1430bc0cec6c4025b812c843d05f0b09 2024-11-19T17:15:12,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/be83d1b679b746e28639269040d78430 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/be83d1b679b746e28639269040d78430 2024-11-19T17:15:12,097 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/8f03d7f0711d43bab4168b9701ed6a26 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/8f03d7f0711d43bab4168b9701ed6a26 2024-11-19T17:15:12,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/b45bd95db0644b4bae11ad1732e91c91 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/b45bd95db0644b4bae11ad1732e91c91 2024-11-19T17:15:12,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/3ba9c918259046a7be5faeda129904e7 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/3ba9c918259046a7be5faeda129904e7 2024-11-19T17:15:12,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/15d205dc6a6c47ada6e303638e1d2b4c to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/15d205dc6a6c47ada6e303638e1d2b4c 2024-11-19T17:15:12,100 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/c8e941e1ef1b48ee86417e78e48e8c76 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/c8e941e1ef1b48ee86417e78e48e8c76 2024-11-19T17:15:12,101 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/2a13b8374a7a4179aac22ff0637b5a51 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/2a13b8374a7a4179aac22ff0637b5a51 2024-11-19T17:15:12,101 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/dcd795898a074edc826256334786ee7b to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/dcd795898a074edc826256334786ee7b 2024-11-19T17:15:12,102 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/d44a8f5f470547c69252713f89a2b558 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/d44a8f5f470547c69252713f89a2b558 2024-11-19T17:15:12,103 DEBUG [StoreCloser-TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/52053f39914a464badb9673089fc9ffc to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/52053f39914a464badb9673089fc9ffc 2024-11-19T17:15:12,107 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/recovered.edits/638.seqid, newMaxSeqId=638, maxSeqId=1 2024-11-19T17:15:12,107 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. 2024-11-19T17:15:12,107 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for c1ce7345860509b228081c672a073b0d: Waiting for close lock at 1732036510797Running coprocessor pre-close hooks at 1732036510797Disabling compacts and flushes for region at 1732036510797Disabling writes for close at 1732036510797Obtaining lock to block concurrent updates at 1732036510797Preparing flush snapshotting stores in c1ce7345860509b228081c672a073b0d at 1732036510797Finished memstore snapshotting TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732036510798 (+1 ms)Flushing stores of TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d. at 1732036510798Flushing c1ce7345860509b228081c672a073b0d/A: creating writer at 1732036510798Flushing c1ce7345860509b228081c672a073b0d/A: appending metadata at 1732036510801 (+3 ms)Flushing c1ce7345860509b228081c672a073b0d/A: closing flushed file at 1732036510801Flushing c1ce7345860509b228081c672a073b0d/B: creating writer at 1732036511209 (+408 ms)Flushing c1ce7345860509b228081c672a073b0d/B: appending metadata at 1732036511211 (+2 ms)Flushing c1ce7345860509b228081c672a073b0d/B: closing flushed file at 1732036511211Flushing c1ce7345860509b228081c672a073b0d/C: creating writer at 1732036511619 (+408 ms)Flushing c1ce7345860509b228081c672a073b0d/C: appending metadata at 1732036511621 (+2 ms)Flushing c1ce7345860509b228081c672a073b0d/C: closing flushed file at 1732036511621Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b090b1b: reopening flushed file at 1732036512028 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5423e755: reopening flushed file at 1732036512032 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48331be2: reopening flushed file at 1732036512035 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c1ce7345860509b228081c672a073b0d in 1242ms, sequenceid=635, compaction requested=true at 1732036512039 (+4 ms)Writing region close event to WAL at 1732036512104 (+65 ms)Running coprocessor post-close hooks at 1732036512107 (+3 ms)Closed at 1732036512107 2024-11-19T17:15:12,109 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed c1ce7345860509b228081c672a073b0d 2024-11-19T17:15:12,109 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=c1ce7345860509b228081c672a073b0d, regionState=CLOSED 2024-11-19T17:15:12,111 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 because future has completed 2024-11-19T17:15:12,113 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-19T17:15:12,113 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure c1ce7345860509b228081c672a073b0d, server=280a42d09ec8,33873,1732036422473 in 1.4670 sec 2024-11-19T17:15:12,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-19T17:15:12,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1ce7345860509b228081c672a073b0d, UNASSIGN in 1.4710 sec 2024-11-19T17:15:12,116 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-19T17:15:12,116 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4740 sec 2024-11-19T17:15:12,117 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732036512117"}]},"ts":"1732036512117"} 2024-11-19T17:15:12,118 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T17:15:12,118 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T17:15:12,119 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4830 sec 2024-11-19T17:15:12,132 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T17:15:12,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-19T17:15:12,784 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:15:12,784 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T17:15:12,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:15:12,786 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=96, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:15:12,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-19T17:15:12,787 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=96, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:15:12,788 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d 2024-11-19T17:15:12,790 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A, FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B, FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C, FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/recovered.edits] 2024-11-19T17:15:12,793 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/8dbc50fa2e07499599ddfdf0beb38779 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/8dbc50fa2e07499599ddfdf0beb38779 2024-11-19T17:15:12,794 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/e205ac54f2634e5dbfe19d2bb3e3013d to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/e205ac54f2634e5dbfe19d2bb3e3013d 2024-11-19T17:15:12,795 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/e9d1afe3ae98409190b3378d92d0fe9e to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/A/e9d1afe3ae98409190b3378d92d0fe9e 2024-11-19T17:15:12,797 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/47c4c0736fbc46c0ba48261cfe07d94d to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/47c4c0736fbc46c0ba48261cfe07d94d 2024-11-19T17:15:12,798 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/d9431356c46e4c8da5bbcfbed2696458 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/d9431356c46e4c8da5bbcfbed2696458 2024-11-19T17:15:12,798 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/eb7939cd96314a02a7c528a80ebe15fc to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/B/eb7939cd96314a02a7c528a80ebe15fc 2024-11-19T17:15:12,800 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/7392662e4e014ca9a262166449d575af to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/7392662e4e014ca9a262166449d575af 2024-11-19T17:15:12,801 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/b53746612fd742189767fc17592c37a4 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/b53746612fd742189767fc17592c37a4 2024-11-19T17:15:12,802 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/e45e8103f6b945a3afea590787166350 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/C/e45e8103f6b945a3afea590787166350 2024-11-19T17:15:12,804 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/recovered.edits/638.seqid to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d/recovered.edits/638.seqid 2024-11-19T17:15:12,805 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/c1ce7345860509b228081c672a073b0d 2024-11-19T17:15:12,805 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T17:15:12,807 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=96, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:15:12,809 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T17:15:12,811 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T17:15:12,812 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=96, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:15:12,812 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T17:15:12,813 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732036512812"}]},"ts":"9223372036854775807"} 2024-11-19T17:15:12,815 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T17:15:12,815 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c1ce7345860509b228081c672a073b0d, NAME => 'TestAcidGuarantees,,1732036486206.c1ce7345860509b228081c672a073b0d.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T17:15:12,815 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T17:15:12,815 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732036512815"}]},"ts":"9223372036854775807"} 2024-11-19T17:15:12,816 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T17:15:12,817 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=96, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:15:12,818 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 33 msec 2024-11-19T17:15:12,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-19T17:15:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T17:15:12,894 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:15:12,903 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-873178183_22 at /127.0.0.1:59826 [Waiting for operation #176] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_569709797_22 at /127.0.0.1:47410 [Waiting for operation #325] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=381 (was 419), ProcessCount=11 (was 11), AvailableMemoryMB=6673 (was 6946) 2024-11-19T17:15:12,912 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=381, ProcessCount=11, AvailableMemoryMB=6673 2024-11-19T17:15:12,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T17:15:12,914 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T17:15:12,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T17:15:12,916 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T17:15:12,916 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:12,916 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 97 2024-11-19T17:15:12,917 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T17:15:12,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T17:15:12,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742211_1387 (size=1061) 2024-11-19T17:15:13,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T17:15:13,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T17:15:13,324 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e 2024-11-19T17:15:13,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742212_1388 (size=53) 2024-11-19T17:15:13,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T17:15:13,730 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:15:13,730 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 086da227605bf568b877f13b452b7f9a, disabling compactions & flushes 2024-11-19T17:15:13,730 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:13,730 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:13,730 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. after waiting 0 ms 2024-11-19T17:15:13,730 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:13,730 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:13,730 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 086da227605bf568b877f13b452b7f9a: Waiting for close lock at 1732036513730Disabling compacts and flushes for region at 1732036513730Disabling writes for close at 1732036513730Writing region close event to WAL at 1732036513730Closed at 1732036513730 2024-11-19T17:15:13,731 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T17:15:13,731 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732036513731"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732036513731"}]},"ts":"1732036513731"} 2024-11-19T17:15:13,733 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T17:15:13,734 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T17:15:13,734 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732036513734"}]},"ts":"1732036513734"} 2024-11-19T17:15:13,735 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T17:15:13,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=086da227605bf568b877f13b452b7f9a, ASSIGN}] 2024-11-19T17:15:13,736 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=086da227605bf568b877f13b452b7f9a, ASSIGN 2024-11-19T17:15:13,737 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=086da227605bf568b877f13b452b7f9a, ASSIGN; state=OFFLINE, location=280a42d09ec8,33873,1732036422473; forceNewPlan=false, retain=false 2024-11-19T17:15:13,887 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=086da227605bf568b877f13b452b7f9a, regionState=OPENING, regionLocation=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:13,889 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=086da227605bf568b877f13b452b7f9a, ASSIGN because future has completed 2024-11-19T17:15:13,889 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473}] 2024-11-19T17:15:14,043 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:14,043 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7752): Opening region: {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:15:14,044 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:14,044 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:15:14,044 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7794): checking encryption for 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:14,044 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7797): checking classloading for 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:14,045 INFO [StoreOpener-086da227605bf568b877f13b452b7f9a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:14,046 INFO [StoreOpener-086da227605bf568b877f13b452b7f9a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T17:15:14,046 INFO [StoreOpener-086da227605bf568b877f13b452b7f9a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 086da227605bf568b877f13b452b7f9a columnFamilyName A 2024-11-19T17:15:14,046 DEBUG [StoreOpener-086da227605bf568b877f13b452b7f9a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:14,047 INFO [StoreOpener-086da227605bf568b877f13b452b7f9a-1 {}] regionserver.HStore(327): Store=086da227605bf568b877f13b452b7f9a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:15:14,047 INFO [StoreOpener-086da227605bf568b877f13b452b7f9a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:14,048 INFO [StoreOpener-086da227605bf568b877f13b452b7f9a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T17:15:14,048 INFO [StoreOpener-086da227605bf568b877f13b452b7f9a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 086da227605bf568b877f13b452b7f9a columnFamilyName B 2024-11-19T17:15:14,048 DEBUG [StoreOpener-086da227605bf568b877f13b452b7f9a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:14,048 INFO [StoreOpener-086da227605bf568b877f13b452b7f9a-1 {}] regionserver.HStore(327): Store=086da227605bf568b877f13b452b7f9a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:15:14,048 INFO [StoreOpener-086da227605bf568b877f13b452b7f9a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:14,049 INFO [StoreOpener-086da227605bf568b877f13b452b7f9a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T17:15:14,049 INFO [StoreOpener-086da227605bf568b877f13b452b7f9a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 086da227605bf568b877f13b452b7f9a columnFamilyName C 2024-11-19T17:15:14,049 DEBUG [StoreOpener-086da227605bf568b877f13b452b7f9a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:14,049 INFO [StoreOpener-086da227605bf568b877f13b452b7f9a-1 {}] regionserver.HStore(327): Store=086da227605bf568b877f13b452b7f9a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:15:14,049 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:14,050 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1038): replaying wal for 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:14,050 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:14,050 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:14,051 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1048): stopping wal replay for 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:14,051 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1060): Cleaning up temporary data for 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:14,051 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T17:15:14,052 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1093): writing seq id for 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T17:15:14,054 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T17:15:14,054 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1114): Opened 086da227605bf568b877f13b452b7f9a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63275866, jitterRate=-0.05711612105369568}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T17:15:14,054 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:14,055 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1006): Region open journal for 086da227605bf568b877f13b452b7f9a: Running coprocessor pre-open hook at 1732036514044Writing region info on filesystem at 1732036514044Initializing all the Stores at 1732036514045 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036514045Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036514045Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036514045Cleaning up temporary data from old regions at 1732036514051 (+6 ms)Running coprocessor post-open hooks at 1732036514054 (+3 ms)Region opened successfully at 1732036514055 (+1 ms) 2024-11-19T17:15:14,055 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., pid=99, masterSystemTime=1732036514041 2024-11-19T17:15:14,057 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:14,057 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:14,058 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=086da227605bf568b877f13b452b7f9a, regionState=OPEN, openSeqNum=2, regionLocation=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:14,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 because future has completed 2024-11-19T17:15:14,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-19T17:15:14,061 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; OpenRegionProcedure 086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 in 170 msec 2024-11-19T17:15:14,062 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-19T17:15:14,062 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=086da227605bf568b877f13b452b7f9a, ASSIGN in 326 msec 2024-11-19T17:15:14,063 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T17:15:14,063 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732036514063"}]},"ts":"1732036514063"} 2024-11-19T17:15:14,064 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T17:15:14,065 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T17:15:14,066 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1510 sec 2024-11-19T17:15:15,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T17:15:15,064 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:15:15,065 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af6f666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:15,065 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:15:15,065 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:15:15,066 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:15:15,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:15:15,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:15:15,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73439d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:15,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:15:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:15:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:15,067 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:15:15,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e405675, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:15,068 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:15:15,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:15:15,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:15:15,070 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:15:15,070 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T17:15:15,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 280a42d09ec8,40603,1732036421715 2024-11-19T17:15:15,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6c685685 2024-11-19T17:15:15,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T17:15:15,072 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T17:15:15,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T17:15:15,073 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T17:15:15,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=100, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T17:15:15,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742213_1389 (size=1097) 2024-11-19T17:15:15,484 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-19T17:15:15,484 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-19T17:15:15,486 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T17:15:15,488 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=086da227605bf568b877f13b452b7f9a, REOPEN/MOVE}] 2024-11-19T17:15:15,489 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=086da227605bf568b877f13b452b7f9a, REOPEN/MOVE 2024-11-19T17:15:15,489 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=086da227605bf568b877f13b452b7f9a, regionState=CLOSING, regionLocation=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:15,491 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=086da227605bf568b877f13b452b7f9a, REOPEN/MOVE because future has completed 2024-11-19T17:15:15,491 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T17:15:15,491 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE, hasLock=false; CloseRegionProcedure 086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473}] 2024-11-19T17:15:15,643 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(122): Close 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:15,643 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T17:15:15,643 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1722): Closing 086da227605bf568b877f13b452b7f9a, disabling compactions & flushes 2024-11-19T17:15:15,643 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:15,643 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:15,643 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. after waiting 0 ms 2024-11-19T17:15:15,643 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:15,646 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T17:15:15,647 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:15,647 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1676): Region close journal for 086da227605bf568b877f13b452b7f9a: Waiting for close lock at 1732036515643Running coprocessor pre-close hooks at 1732036515643Disabling compacts and flushes for region at 1732036515643Disabling writes for close at 1732036515643Writing region close event to WAL at 1732036515643Running coprocessor post-close hooks at 1732036515647 (+4 ms)Closed at 1732036515647 2024-11-19T17:15:15,647 WARN [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegionServer(3299): Not adding moved region record: 086da227605bf568b877f13b452b7f9a to self. 2024-11-19T17:15:15,648 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(157): Closed 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:15,649 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=086da227605bf568b877f13b452b7f9a, regionState=CLOSED 2024-11-19T17:15:15,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=103, ppid=102, state=RUNNABLE, hasLock=false; CloseRegionProcedure 086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 because future has completed 2024-11-19T17:15:15,652 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-11-19T17:15:15,652 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; CloseRegionProcedure 086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 in 160 msec 2024-11-19T17:15:15,652 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=086da227605bf568b877f13b452b7f9a, REOPEN/MOVE; state=CLOSED, location=280a42d09ec8,33873,1732036422473; forceNewPlan=false, retain=true 2024-11-19T17:15:15,803 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=086da227605bf568b877f13b452b7f9a, regionState=OPENING, regionLocation=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:15,804 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=086da227605bf568b877f13b452b7f9a, REOPEN/MOVE because future has completed 2024-11-19T17:15:15,804 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=102, state=RUNNABLE, hasLock=false; OpenRegionProcedure 086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473}] 2024-11-19T17:15:15,959 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:15,959 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7752): Opening region: {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:15:15,959 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:15,959 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:15:15,959 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7794): checking encryption for 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:15,959 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7797): checking classloading for 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:15,960 INFO [StoreOpener-086da227605bf568b877f13b452b7f9a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:15,961 INFO [StoreOpener-086da227605bf568b877f13b452b7f9a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T17:15:15,961 INFO [StoreOpener-086da227605bf568b877f13b452b7f9a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 086da227605bf568b877f13b452b7f9a columnFamilyName A 2024-11-19T17:15:15,962 DEBUG [StoreOpener-086da227605bf568b877f13b452b7f9a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:15,962 INFO [StoreOpener-086da227605bf568b877f13b452b7f9a-1 {}] regionserver.HStore(327): Store=086da227605bf568b877f13b452b7f9a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:15:15,963 INFO [StoreOpener-086da227605bf568b877f13b452b7f9a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:15,963 INFO [StoreOpener-086da227605bf568b877f13b452b7f9a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T17:15:15,964 INFO [StoreOpener-086da227605bf568b877f13b452b7f9a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 086da227605bf568b877f13b452b7f9a columnFamilyName B 2024-11-19T17:15:15,964 DEBUG [StoreOpener-086da227605bf568b877f13b452b7f9a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:15,964 INFO [StoreOpener-086da227605bf568b877f13b452b7f9a-1 {}] regionserver.HStore(327): Store=086da227605bf568b877f13b452b7f9a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:15:15,964 INFO [StoreOpener-086da227605bf568b877f13b452b7f9a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:15,964 INFO [StoreOpener-086da227605bf568b877f13b452b7f9a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T17:15:15,965 INFO [StoreOpener-086da227605bf568b877f13b452b7f9a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 086da227605bf568b877f13b452b7f9a columnFamilyName C 2024-11-19T17:15:15,965 DEBUG [StoreOpener-086da227605bf568b877f13b452b7f9a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:15,965 INFO [StoreOpener-086da227605bf568b877f13b452b7f9a-1 {}] regionserver.HStore(327): Store=086da227605bf568b877f13b452b7f9a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:15:15,965 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:15,965 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1038): replaying wal for 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:15,966 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:15,966 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:15,967 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1048): stopping wal replay for 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:15,967 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1060): Cleaning up temporary data for 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:15,967 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T17:15:15,968 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1093): writing seq id for 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:15,969 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1114): Opened 086da227605bf568b877f13b452b7f9a; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64852284, jitterRate=-0.033625662326812744}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T17:15:15,969 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:15,969 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1006): Region open journal for 086da227605bf568b877f13b452b7f9a: Running coprocessor pre-open hook at 1732036515959Writing region info on filesystem at 1732036515959Initializing all the Stores at 1732036515960 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036515960Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036515960Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036515960Cleaning up temporary data from old regions at 1732036515967 (+7 ms)Running coprocessor post-open hooks at 1732036515969 (+2 ms)Region opened successfully at 1732036515969 2024-11-19T17:15:15,970 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., pid=104, masterSystemTime=1732036515956 2024-11-19T17:15:15,972 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:15,972 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:15,972 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=086da227605bf568b877f13b452b7f9a, regionState=OPEN, openSeqNum=5, regionLocation=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:15,973 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=102, state=RUNNABLE, hasLock=false; OpenRegionProcedure 086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 because future has completed 2024-11-19T17:15:15,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=102 2024-11-19T17:15:15,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=102, state=SUCCESS, hasLock=false; OpenRegionProcedure 086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 in 170 msec 2024-11-19T17:15:15,976 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-19T17:15:15,976 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=086da227605bf568b877f13b452b7f9a, REOPEN/MOVE in 487 msec 2024-11-19T17:15:15,978 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-11-19T17:15:15,978 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 491 msec 2024-11-19T17:15:15,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 906 msec 2024-11-19T17:15:15,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=100 2024-11-19T17:15:15,981 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:15:15,982 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ff38b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:15,982 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:15:15,982 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:15:15,983 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:15:15,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:15:15,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:15:15,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a4d3369, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:15,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:15:15,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:15:15,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:15,984 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:15:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc9164f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:15,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ec6412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:15,985 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:15:15,985 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:15:15,986 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:15:15,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:15:15,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:15:15,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e26887d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:15,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:15:15,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:15:15,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:15,987 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:15:15,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@674e95b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:15,988 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f2378c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:15,988 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:15:15,988 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:15:15,989 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:15:15,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:15:15,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:15:15,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a1cf0ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:15,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:15:15,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:15:15,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:15,990 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:15:15,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76f4b068, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:15,991 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cc49f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:15,991 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:15:15,991 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:15:15,992 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:15:15,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:15:15,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:15:15,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a76aab3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:15,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:15:15,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:15:15,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:15,993 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:15:15,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@177d8eae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:15,994 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c86b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:15,994 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:15:15,994 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:15:15,995 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:15:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:15:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:15:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625724d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:15:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:15:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:15,996 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:15:15,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64c151d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:15,997 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d098cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:15,997 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:15:15,997 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:15:15,997 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:15:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:15:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:15:15,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42ced3d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:15,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:15:15,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:15:15,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:15,998 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:15:15,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5643b29d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:15,999 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c9deaeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:15,999 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:15:15,999 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:15:16,000 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:15:16,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:15:16,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:15:16,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46e7bcbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:16,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:15:16,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:15:16,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:16,001 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:15:16,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f77d72b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:16,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f8ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:16,002 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:15:16,002 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:15:16,003 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:15:16,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:15:16,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:15:16,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@638a931, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:16,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:15:16,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:15:16,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:16,004 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:15:16,004 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f5f66b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:16,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1515ed5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:16,005 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:15:16,005 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:15:16,005 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:15:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:15:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:15:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79e014a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:15:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:15:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:16,007 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:15:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555bfcba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:16,008 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d10c554, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:16,008 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:15:16,008 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:15:16,009 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:15:16,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:15:16,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:15:16,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41166a71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:16,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:15:16,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:15:16,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:16,010 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:15:16,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e5df9dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:16,011 DEBUG [Thread-1747 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:15:16,011 DEBUG [Thread-1743 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:15:16,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:15:16,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:15:16,012 DEBUG [Thread-1756 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:15:16,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:15:16,012 DEBUG [Thread-1758 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:15:16,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:15:16,013 DEBUG [Thread-1751 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:15:16,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:15:16,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:15:16,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:15:16,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:15:16,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:15:16,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:15:16,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:15:16,014 DEBUG [Thread-1754 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:15:16,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:15:16,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:15:16,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:15:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:15:16,015 DEBUG [Thread-1762 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:15:16,015 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:15:16,015 DEBUG [Thread-1754 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:15:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:15:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:15:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5] 2024-11-19T17:15:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5] 2024-11-19T17:15:16,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5] 2024-11-19T17:15:16,017 DEBUG [Thread-1749 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:15:16,017 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:15:16,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5] 2024-11-19T17:15:16,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5] 2024-11-19T17:15:16,018 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:15:16,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:15:16,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:15:16,019 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:15:16,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5] 2024-11-19T17:15:16,021 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:15:16,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5] 2024-11-19T17:15:16,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5] 2024-11-19T17:15:16,022 DEBUG [Thread-1760 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:15:16,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-19T17:15:16,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T17:15:16,024 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:15:16,024 DEBUG [Thread-1745 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:15:16,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:15:16,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:15:16,026 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:15:16,026 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:15:16,027 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:15:16,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5] 2024-11-19T17:15:16,030 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:15:16,030 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:15:16,032 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:15:16,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 086da227605bf568b877f13b452b7f9a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T17:15:16,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:16,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=A 2024-11-19T17:15:16,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:16,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=B 2024-11-19T17:15:16,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:16,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=C 2024-11-19T17:15:16,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:16,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5] 2024-11-19T17:15:16,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119138864a3dab541b4bc43e6f70bfcd938_086da227605bf568b877f13b452b7f9a is 50, key is test_row_0/A:col10/1732036516032/Put/seqid=0 2024-11-19T17:15:16,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742214_1390 (size=17034) 2024-11-19T17:15:16,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T17:15:16,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036576139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:16,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:16,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036576142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:16,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:16,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036576143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:16,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036516137, max timestamp=1732036516154Num uniques 859; 2024-11-19T17:15:16,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036516137, max timestamp=1732036516154Num uniques 859; 2024-11-19T17:15:16,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:15:16,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:15:16,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036516137, max timestamp=1732036516154Num uniques 859; 2024-11-19T17:15:16,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:15:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:16,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:16,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:16,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:16,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:16,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T17:15:16,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:16,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:16,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:16,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:16,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:16,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036576252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:16,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:16,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036576259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:16,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:16,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:16,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:16,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:16,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036576264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:16,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:16,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:16,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:16,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:16,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036576264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:16,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:16,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036576264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:16,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:16,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:16,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:16,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:16,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:16,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:16,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T17:15:16,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:16,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:16,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:16,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:16,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:16,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:16,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T17:15:16,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:16,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036576384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:16,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036576384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:16,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:16,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:16,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:16,461 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,467 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119138864a3dab541b4bc43e6f70bfcd938_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119138864a3dab541b4bc43e6f70bfcd938_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:16,468 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/845c360d9cee4d4985c37191abfde3e0, store: [table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:16,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/845c360d9cee4d4985c37191abfde3e0 is 175, key is test_row_0/A:col10/1732036516032/Put/seqid=0 2024-11-19T17:15:16,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742215_1391 (size=48139) 2024-11-19T17:15:16,476 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/845c360d9cee4d4985c37191abfde3e0 2024-11-19T17:15:16,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:16,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036576483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:16,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:16,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036576483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:16,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:16,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036576484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:16,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T17:15:16,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:16,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:16,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:16,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:16,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:16,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:16,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/fbd23b0c49494ba5b2c1f7f2edef84b4 is 50, key is test_row_0/B:col10/1732036516032/Put/seqid=0 2024-11-19T17:15:16,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742216_1392 (size=12001) 2024-11-19T17:15:16,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/fbd23b0c49494ba5b2c1f7f2edef84b4 2024-11-19T17:15:16,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/0fc3ac0534584d1fbcbd133ca9730388 is 50, key is test_row_0/C:col10/1732036516032/Put/seqid=0 2024-11-19T17:15:16,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:16,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036576603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:16,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:16,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036576603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:16,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742217_1393 (size=12001) 2024-11-19T17:15:16,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:16,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:16,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:16,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/0fc3ac0534584d1fbcbd133ca9730388 2024-11-19T17:15:16,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/845c360d9cee4d4985c37191abfde3e0 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/845c360d9cee4d4985c37191abfde3e0 2024-11-19T17:15:16,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/845c360d9cee4d4985c37191abfde3e0, entries=250, sequenceid=16, filesize=47.0 K 2024-11-19T17:15:16,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/fbd23b0c49494ba5b2c1f7f2edef84b4 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/fbd23b0c49494ba5b2c1f7f2edef84b4 2024-11-19T17:15:16,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/fbd23b0c49494ba5b2c1f7f2edef84b4, entries=150, sequenceid=16, filesize=11.7 K 2024-11-19T17:15:16,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/0fc3ac0534584d1fbcbd133ca9730388 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/0fc3ac0534584d1fbcbd133ca9730388 2024-11-19T17:15:16,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/0fc3ac0534584d1fbcbd133ca9730388, entries=150, sequenceid=16, filesize=11.7 K 2024-11-19T17:15:16,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 086da227605bf568b877f13b452b7f9a in 615ms, sequenceid=16, compaction requested=false 2024-11-19T17:15:16,647 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T17:15:16,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:16,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T17:15:16,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:16,652 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 086da227605bf568b877f13b452b7f9a 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-11-19T17:15:16,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=A 2024-11-19T17:15:16,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:16,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=B 2024-11-19T17:15:16,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:16,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=C 2024-11-19T17:15:16,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:16,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T17:15:16,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111938a15c8d2047488dbd9f5d510ab86e48_086da227605bf568b877f13b452b7f9a is 50, key is test_row_0/A:col10/1732036516240/Put/seqid=0 2024-11-19T17:15:16,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742218_1394 (size=12154) 2024-11-19T17:15:16,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,682 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111938a15c8d2047488dbd9f5d510ab86e48_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111938a15c8d2047488dbd9f5d510ab86e48_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:16,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/b3bc6c4525e44c559357f5e1126dae4a, store: [table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:16,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/b3bc6c4525e44c559357f5e1126dae4a is 175, key is test_row_0/A:col10/1732036516240/Put/seqid=0 2024-11-19T17:15:16,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742219_1395 (size=30955) 2024-11-19T17:15:16,707 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/b3bc6c4525e44c559357f5e1126dae4a 2024-11-19T17:15:16,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/1d8d8f2416c2447494f30e214c2c64cb is 50, key is test_row_0/B:col10/1732036516240/Put/seqid=0 2024-11-19T17:15:16,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742220_1396 (size=12001) 2024-11-19T17:15:16,737 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/1d8d8f2416c2447494f30e214c2c64cb 2024-11-19T17:15:16,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/34f6e1e3741b45eeb7d64631e4565897 is 50, key is test_row_0/C:col10/1732036516240/Put/seqid=0 2024-11-19T17:15:16,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742221_1397 (size=12001) 2024-11-19T17:15:16,769 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/34f6e1e3741b45eeb7d64631e4565897 2024-11-19T17:15:16,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/b3bc6c4525e44c559357f5e1126dae4a as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/b3bc6c4525e44c559357f5e1126dae4a 2024-11-19T17:15:16,783 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/b3bc6c4525e44c559357f5e1126dae4a, entries=150, sequenceid=50, filesize=30.2 K 2024-11-19T17:15:16,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/1d8d8f2416c2447494f30e214c2c64cb as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/1d8d8f2416c2447494f30e214c2c64cb 2024-11-19T17:15:16,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,788 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/1d8d8f2416c2447494f30e214c2c64cb, entries=150, sequenceid=50, filesize=11.7 K 2024-11-19T17:15:16,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/34f6e1e3741b45eeb7d64631e4565897 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/34f6e1e3741b45eeb7d64631e4565897 2024-11-19T17:15:16,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,793 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/34f6e1e3741b45eeb7d64631e4565897, entries=150, sequenceid=50, filesize=11.7 K 2024-11-19T17:15:16,794 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=0 B/0 for 086da227605bf568b877f13b452b7f9a in 142ms, sequenceid=50, compaction requested=false 2024-11-19T17:15:16,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:16,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:16,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-19T17:15:16,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-19T17:15:16,798 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-19T17:15:16,798 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 770 msec 2024-11-19T17:15:16,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 778 msec 2024-11-19T17:15:16,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:16,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,042 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 086da227605bf568b877f13b452b7f9a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T17:15:17,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=A 2024-11-19T17:15:17,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:17,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=B 2024-11-19T17:15:17,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:17,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=C 2024-11-19T17:15:17,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:17,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d113fedc96af4a708fda84e6c5d2dbd9_086da227605bf568b877f13b452b7f9a is 50, key is test_row_0/A:col10/1732036516990/Put/seqid=0 2024-11-19T17:15:17,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742222_1398 (size=21918) 2024-11-19T17:15:17,134 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:17,141 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d113fedc96af4a708fda84e6c5d2dbd9_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d113fedc96af4a708fda84e6c5d2dbd9_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:17,143 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/65b085eb43484f509735372e871e58ec, store: [table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:17,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/65b085eb43484f509735372e871e58ec is 175, key is test_row_0/A:col10/1732036516990/Put/seqid=0 2024-11-19T17:15:17,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T17:15:17,164 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:15:17,164 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:15:17,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-19T17:15:17,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T17:15:17,166 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:15:17,167 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:15:17,167 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:15:17,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732036517115, max timestamp=1732036517174Num uniques 859; 2024-11-19T17:15:17,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732036517115, max timestamp=1732036517174Num uniques 859; 2024-11-19T17:15:17,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732036517115, max timestamp=1732036517174Num uniques 859; 2024-11-19T17:15:17,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T17:15:17,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T17:15:17,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T17:15:17,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742223_1399 (size=65323) 2024-11-19T17:15:17,199 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/65b085eb43484f509735372e871e58ec 2024-11-19T17:15:17,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/fbe9652e4c40438082d1ff4cf6864369 is 50, key is test_row_0/B:col10/1732036516990/Put/seqid=0 2024-11-19T17:15:17,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742224_1400 (size=12001) 2024-11-19T17:15:17,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:17,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036577227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:17,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:17,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:17,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036577227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:17,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:17,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036577229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:17,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:17,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T17:15:17,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:17,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036577253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:17,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:17,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036577256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:17,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:17,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:17,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T17:15:17,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:17,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:17,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:17,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:17,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:17,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:17,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:17,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036577364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:17,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:17,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036577365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:17,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:17,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:17,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036577365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:17,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:17,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:17,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:17,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036577384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:17,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:17,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036577384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:17,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:17,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:17,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T17:15:17,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:17,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:17,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:17,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:17,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:17,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:17,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T17:15:17,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:17,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036577585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:17,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:17,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036577585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:17,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:17,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:17,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036577585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:17,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:17,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:17,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:17,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036577604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:17,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:17,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036577604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:17,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:17,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:17,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T17:15:17,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:17,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:17,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:17,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:17,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:17,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:17,649 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/fbe9652e4c40438082d1ff4cf6864369 2024-11-19T17:15:17,677 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T17:15:17,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/86cb26ae4caf45a1a894f02fd4d9219b is 50, key is test_row_0/C:col10/1732036516990/Put/seqid=0 2024-11-19T17:15:17,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742225_1401 (size=12001) 2024-11-19T17:15:17,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T17:15:17,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:17,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:17,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:17,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:17,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:17,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:17,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T17:15:17,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:17,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036577913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:17,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:17,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:17,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036577914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:17,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:17,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036577914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:17,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:17,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:17,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T17:15:17,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:17,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:17,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:17,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:17,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:17,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036577925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:17,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036577926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:18,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T17:15:18,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:18,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:18,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:18,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:18,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:18,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/86cb26ae4caf45a1a894f02fd4d9219b 2024-11-19T17:15:18,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/65b085eb43484f509735372e871e58ec as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/65b085eb43484f509735372e871e58ec 2024-11-19T17:15:18,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/65b085eb43484f509735372e871e58ec, entries=350, sequenceid=60, filesize=63.8 K 2024-11-19T17:15:18,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/fbe9652e4c40438082d1ff4cf6864369 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/fbe9652e4c40438082d1ff4cf6864369 2024-11-19T17:15:18,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/fbe9652e4c40438082d1ff4cf6864369, entries=150, sequenceid=60, filesize=11.7 K 2024-11-19T17:15:18,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/86cb26ae4caf45a1a894f02fd4d9219b as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/86cb26ae4caf45a1a894f02fd4d9219b 2024-11-19T17:15:18,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/86cb26ae4caf45a1a894f02fd4d9219b, entries=150, sequenceid=60, filesize=11.7 K 2024-11-19T17:15:18,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 086da227605bf568b877f13b452b7f9a in 1123ms, sequenceid=60, compaction requested=true 2024-11-19T17:15:18,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:18,165 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:18,165 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 144417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:18,165 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 086da227605bf568b877f13b452b7f9a/A is initiating minor compaction (all files) 2024-11-19T17:15:18,166 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 086da227605bf568b877f13b452b7f9a/A in TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:18,166 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/845c360d9cee4d4985c37191abfde3e0, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/b3bc6c4525e44c559357f5e1126dae4a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/65b085eb43484f509735372e871e58ec] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp, totalSize=141.0 K 2024-11-19T17:15:18,166 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:18,166 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. files: [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/845c360d9cee4d4985c37191abfde3e0, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/b3bc6c4525e44c559357f5e1126dae4a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/65b085eb43484f509735372e871e58ec] 2024-11-19T17:15:18,166 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 845c360d9cee4d4985c37191abfde3e0, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732036516025 2024-11-19T17:15:18,166 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3bc6c4525e44c559357f5e1126dae4a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732036516240 2024-11-19T17:15:18,167 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65b085eb43484f509735372e871e58ec, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732036516932 2024-11-19T17:15:18,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 086da227605bf568b877f13b452b7f9a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:15:18,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:18,167 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:18,168 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:18,168 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 086da227605bf568b877f13b452b7f9a/B is initiating minor compaction (all files) 2024-11-19T17:15:18,168 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 086da227605bf568b877f13b452b7f9a/B in TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:18,168 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/fbd23b0c49494ba5b2c1f7f2edef84b4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/1d8d8f2416c2447494f30e214c2c64cb, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/fbe9652e4c40438082d1ff4cf6864369] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp, totalSize=35.2 K 2024-11-19T17:15:18,169 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting fbd23b0c49494ba5b2c1f7f2edef84b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732036516027 2024-11-19T17:15:18,169 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d8d8f2416c2447494f30e214c2c64cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732036516240 2024-11-19T17:15:18,170 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting fbe9652e4c40438082d1ff4cf6864369, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732036516990 2024-11-19T17:15:18,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 086da227605bf568b877f13b452b7f9a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:15:18,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:18,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 086da227605bf568b877f13b452b7f9a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:15:18,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:18,181 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:18,193 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 086da227605bf568b877f13b452b7f9a#B#compaction#344 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:18,195 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/66afa27d971b4a758dfedaf8c69453ec is 50, key is test_row_0/B:col10/1732036516990/Put/seqid=0 2024-11-19T17:15:18,206 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119bca95bfd19de4e8cbebfe7b8c86892c3_086da227605bf568b877f13b452b7f9a store=[table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:18,209 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119bca95bfd19de4e8cbebfe7b8c86892c3_086da227605bf568b877f13b452b7f9a, store=[table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:18,209 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119bca95bfd19de4e8cbebfe7b8c86892c3_086da227605bf568b877f13b452b7f9a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:18,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T17:15:18,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:18,245 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 086da227605bf568b877f13b452b7f9a 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-19T17:15:18,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=A 2024-11-19T17:15:18,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:18,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=B 2024-11-19T17:15:18,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:18,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=C 2024-11-19T17:15:18,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:18,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742226_1402 (size=12104) 2024-11-19T17:15:18,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742227_1403 (size=4469) 2024-11-19T17:15:18,289 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 086da227605bf568b877f13b452b7f9a#A#compaction#343 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:18,290 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/1d4d80b2b4904d6498731ddb87e6a342 is 175, key is test_row_0/A:col10/1732036516990/Put/seqid=0 2024-11-19T17:15:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T17:15:18,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f4e647c3be7b4bdb94e48e113cb761cd_086da227605bf568b877f13b452b7f9a is 50, key is test_row_0/A:col10/1732036517224/Put/seqid=0 2024-11-19T17:15:18,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742228_1404 (size=31058) 2024-11-19T17:15:18,379 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/1d4d80b2b4904d6498731ddb87e6a342 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/1d4d80b2b4904d6498731ddb87e6a342 2024-11-19T17:15:18,384 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 086da227605bf568b877f13b452b7f9a/A of 086da227605bf568b877f13b452b7f9a into 1d4d80b2b4904d6498731ddb87e6a342(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:18,384 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:18,384 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., storeName=086da227605bf568b877f13b452b7f9a/A, priority=13, startTime=1732036518164; duration=0sec 2024-11-19T17:15:18,384 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:18,385 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 086da227605bf568b877f13b452b7f9a:A 2024-11-19T17:15:18,385 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:18,385 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:18,386 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 086da227605bf568b877f13b452b7f9a/C is initiating minor compaction (all files) 2024-11-19T17:15:18,386 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 086da227605bf568b877f13b452b7f9a/C in TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:18,386 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/0fc3ac0534584d1fbcbd133ca9730388, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/34f6e1e3741b45eeb7d64631e4565897, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/86cb26ae4caf45a1a894f02fd4d9219b] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp, totalSize=35.2 K 2024-11-19T17:15:18,386 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fc3ac0534584d1fbcbd133ca9730388, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732036516027 2024-11-19T17:15:18,386 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34f6e1e3741b45eeb7d64631e4565897, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732036516240 2024-11-19T17:15:18,387 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86cb26ae4caf45a1a894f02fd4d9219b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732036516990 2024-11-19T17:15:18,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742229_1405 (size=12154) 2024-11-19T17:15:18,419 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 086da227605bf568b877f13b452b7f9a#C#compaction#346 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:18,420 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/6f808c493e0e442e8961762552bed051 is 50, key is test_row_0/C:col10/1732036516990/Put/seqid=0 2024-11-19T17:15:18,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:18,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:18,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742230_1406 (size=12104) 2024-11-19T17:15:18,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:18,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036578472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:18,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:18,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:18,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:18,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036578474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:18,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:18,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:18,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:18,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:18,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:18,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036578475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:18,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:18,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036578482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:18,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:18,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:18,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:18,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:18,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036578482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:18,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:18,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:18,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:18,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:18,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036578604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:18,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:18,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:18,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036578604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:18,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:18,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:18,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:18,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:18,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:18,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:18,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036578614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:18,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:18,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:18,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:18,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:18,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036578614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:18,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:18,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:18,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:18,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036578614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:18,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:18,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:18,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:18,672 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/66afa27d971b4a758dfedaf8c69453ec as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/66afa27d971b4a758dfedaf8c69453ec 2024-11-19T17:15:18,677 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 086da227605bf568b877f13b452b7f9a/B of 086da227605bf568b877f13b452b7f9a into 66afa27d971b4a758dfedaf8c69453ec(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:18,677 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:18,677 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., storeName=086da227605bf568b877f13b452b7f9a/B, priority=13, startTime=1732036518167; duration=0sec 2024-11-19T17:15:18,677 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:18,677 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 086da227605bf568b877f13b452b7f9a:B 2024-11-19T17:15:18,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:18,805 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f4e647c3be7b4bdb94e48e113cb761cd_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f4e647c3be7b4bdb94e48e113cb761cd_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:18,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/e638c055f90142b497eb27b142bd3b94, store: [table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:18,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/e638c055f90142b497eb27b142bd3b94 is 175, key is test_row_0/A:col10/1732036517224/Put/seqid=0 2024-11-19T17:15:18,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:18,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036578814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:18,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:18,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:18,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:18,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036578814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:18,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:18,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:18,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:18,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:18,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036578834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:18,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036578834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:18,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:18,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:18,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:18,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:18,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036578834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:18,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:18,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:18,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:18,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742231_1407 (size=30955) 2024-11-19T17:15:18,881 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/6f808c493e0e442e8961762552bed051 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/6f808c493e0e442e8961762552bed051 2024-11-19T17:15:18,892 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 086da227605bf568b877f13b452b7f9a/C of 086da227605bf568b877f13b452b7f9a into 6f808c493e0e442e8961762552bed051(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:18,892 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:18,892 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., storeName=086da227605bf568b877f13b452b7f9a/C, priority=13, startTime=1732036518177; duration=0sec 2024-11-19T17:15:18,892 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:18,892 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 086da227605bf568b877f13b452b7f9a:C 2024-11-19T17:15:19,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:19,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036579134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:19,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:19,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036579134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:19,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:19,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:19,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:19,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:19,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:19,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:19,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:19,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036579153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:19,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036579154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:19,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036579154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:19,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:19,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:19,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:19,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:19,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:19,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:19,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:19,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:19,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:19,251 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/e638c055f90142b497eb27b142bd3b94 2024-11-19T17:15:19,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/9170c2d60f87451d8f4780b38b419e9d is 50, key is test_row_0/B:col10/1732036517224/Put/seqid=0 2024-11-19T17:15:19,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742232_1408 (size=12001) 2024-11-19T17:15:19,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T17:15:19,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:19,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036579653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:19,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:19,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036579654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:19,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:19,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:19,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:19,662 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/9170c2d60f87451d8f4780b38b419e9d 2024-11-19T17:15:19,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/c67e6f9ca7344a4eb621bc734e502ed3 is 50, key is test_row_0/C:col10/1732036517224/Put/seqid=0 2024-11-19T17:15:19,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742233_1409 (size=12001) 2024-11-19T17:15:19,677 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/c67e6f9ca7344a4eb621bc734e502ed3 2024-11-19T17:15:19,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/e638c055f90142b497eb27b142bd3b94 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/e638c055f90142b497eb27b142bd3b94 2024-11-19T17:15:19,685 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/e638c055f90142b497eb27b142bd3b94, entries=150, sequenceid=94, filesize=30.2 K 2024-11-19T17:15:19,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/9170c2d60f87451d8f4780b38b419e9d as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/9170c2d60f87451d8f4780b38b419e9d 2024-11-19T17:15:19,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:19,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036579674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:19,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:19,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036579676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:19,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:19,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036579676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:19,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:19,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:19,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:19,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:19,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:19,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:19,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:19,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:19,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:19,691 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/9170c2d60f87451d8f4780b38b419e9d, entries=150, sequenceid=94, filesize=11.7 K 2024-11-19T17:15:19,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/c67e6f9ca7344a4eb621bc734e502ed3 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/c67e6f9ca7344a4eb621bc734e502ed3 2024-11-19T17:15:19,695 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/c67e6f9ca7344a4eb621bc734e502ed3, entries=150, sequenceid=94, filesize=11.7 K 2024-11-19T17:15:19,696 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=53.67 KB/54960 for 086da227605bf568b877f13b452b7f9a in 1451ms, sequenceid=94, compaction requested=false 2024-11-19T17:15:19,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:19,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:19,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-19T17:15:19,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-19T17:15:19,700 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-19T17:15:19,700 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5300 sec 2024-11-19T17:15:19,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.5370 sec 2024-11-19T17:15:20,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:20,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 086da227605bf568b877f13b452b7f9a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T17:15:20,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=A 2024-11-19T17:15:20,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:20,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=B 2024-11-19T17:15:20,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:20,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=C 2024-11-19T17:15:20,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:20,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c0b515e804c24839bc34145f35fae8a4_086da227605bf568b877f13b452b7f9a is 50, key is test_row_0/A:col10/1732036520669/Put/seqid=0 2024-11-19T17:15:20,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742234_1410 (size=14594) 2024-11-19T17:15:20,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:20,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036580734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:20,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:20,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036580734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:20,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:20,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036580735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:20,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:20,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036580746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:20,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:20,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036580747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:20,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:20,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036580864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:20,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:20,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036580864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:20,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:20,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:20,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:20,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:20,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:20,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:20,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036580864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:20,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:20,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:20,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:20,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036580864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:20,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:20,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:20,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:20,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:20,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036580864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:20,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:20,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:20,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:20,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:20,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:21,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:21,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036581074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:21,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:21,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036581074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:21,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:21,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036581074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:21,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:21,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:21,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036581074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:21,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:21,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:21,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:21,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:21,087 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:21,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:21,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036581083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:21,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:21,090 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c0b515e804c24839bc34145f35fae8a4_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c0b515e804c24839bc34145f35fae8a4_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:21,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:21,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:21,091 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/0537babac0f84579a70afac6728889b7, store: [table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:21,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/0537babac0f84579a70afac6728889b7 is 175, key is test_row_0/A:col10/1732036520669/Put/seqid=0 2024-11-19T17:15:21,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742235_1411 (size=39549) 2024-11-19T17:15:21,096 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=109, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/0537babac0f84579a70afac6728889b7 2024-11-19T17:15:21,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/fdc7481dbd014bf6bbe74e814768936f is 50, key is test_row_0/B:col10/1732036520669/Put/seqid=0 2024-11-19T17:15:21,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742236_1412 (size=12001) 2024-11-19T17:15:21,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T17:15:21,334 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:15:21,334 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:15:21,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-19T17:15:21,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T17:15:21,336 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:15:21,336 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:15:21,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:15:21,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036581394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:21,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036581394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:21,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:21,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:21,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:21,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:21,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:21,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:21,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036581394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:21,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:21,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036581394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:21,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:21,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:21,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:21,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:21,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:21,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:21,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036581403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:21,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:21,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:21,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:21,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T17:15:21,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-19T17:15:21,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:21,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:21,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:21,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:21,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:21,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:21,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/fdc7481dbd014bf6bbe74e814768936f 2024-11-19T17:15:21,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/761c5122ad4b412d82d0efcb133c6903 is 50, key is test_row_0/C:col10/1732036520669/Put/seqid=0 2024-11-19T17:15:21,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742237_1413 (size=12001) 2024-11-19T17:15:21,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/761c5122ad4b412d82d0efcb133c6903 2024-11-19T17:15:21,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/0537babac0f84579a70afac6728889b7 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/0537babac0f84579a70afac6728889b7 2024-11-19T17:15:21,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/0537babac0f84579a70afac6728889b7, entries=200, sequenceid=109, filesize=38.6 K 2024-11-19T17:15:21,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/fdc7481dbd014bf6bbe74e814768936f as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/fdc7481dbd014bf6bbe74e814768936f 2024-11-19T17:15:21,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/fdc7481dbd014bf6bbe74e814768936f, entries=150, sequenceid=109, filesize=11.7 K 2024-11-19T17:15:21,545 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-19T17:15:21,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/761c5122ad4b412d82d0efcb133c6903 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/761c5122ad4b412d82d0efcb133c6903 2024-11-19T17:15:21,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/761c5122ad4b412d82d0efcb133c6903, entries=150, sequenceid=109, filesize=11.7 K 2024-11-19T17:15:21,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 086da227605bf568b877f13b452b7f9a in 881ms, sequenceid=109, compaction requested=true 2024-11-19T17:15:21,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:21,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 086da227605bf568b877f13b452b7f9a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:15:21,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:21,552 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:21,552 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:21,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 086da227605bf568b877f13b452b7f9a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:15:21,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:21,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 086da227605bf568b877f13b452b7f9a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:15:21,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:21,553 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:21,553 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:21,553 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 086da227605bf568b877f13b452b7f9a/A is initiating minor compaction (all files) 2024-11-19T17:15:21,553 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 086da227605bf568b877f13b452b7f9a/B is initiating minor compaction (all files) 2024-11-19T17:15:21,553 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 086da227605bf568b877f13b452b7f9a/A in TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:21,553 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 086da227605bf568b877f13b452b7f9a/B in TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:21,553 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/1d4d80b2b4904d6498731ddb87e6a342, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/e638c055f90142b497eb27b142bd3b94, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/0537babac0f84579a70afac6728889b7] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp, totalSize=99.2 K 2024-11-19T17:15:21,553 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/66afa27d971b4a758dfedaf8c69453ec, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/9170c2d60f87451d8f4780b38b419e9d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/fdc7481dbd014bf6bbe74e814768936f] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp, totalSize=35.3 K 2024-11-19T17:15:21,553 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:21,553 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. files: [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/1d4d80b2b4904d6498731ddb87e6a342, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/e638c055f90142b497eb27b142bd3b94, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/0537babac0f84579a70afac6728889b7] 2024-11-19T17:15:21,554 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 66afa27d971b4a758dfedaf8c69453ec, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732036516990 2024-11-19T17:15:21,554 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d4d80b2b4904d6498731ddb87e6a342, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732036516990 2024-11-19T17:15:21,554 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting e638c055f90142b497eb27b142bd3b94, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732036517224 2024-11-19T17:15:21,554 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 9170c2d60f87451d8f4780b38b419e9d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732036517224 2024-11-19T17:15:21,554 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0537babac0f84579a70afac6728889b7, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732036518473 2024-11-19T17:15:21,554 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting fdc7481dbd014bf6bbe74e814768936f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732036518473 2024-11-19T17:15:21,562 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:21,562 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 086da227605bf568b877f13b452b7f9a#B#compaction#352 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:21,563 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/ddb76d7d087d4623a91d1757f72d0fff is 50, key is test_row_0/B:col10/1732036520669/Put/seqid=0 2024-11-19T17:15:21,564 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111917d75462536640d89d5a3a51c7d5a3da_086da227605bf568b877f13b452b7f9a store=[table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:21,567 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111917d75462536640d89d5a3a51c7d5a3da_086da227605bf568b877f13b452b7f9a, store=[table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:21,567 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111917d75462536640d89d5a3a51c7d5a3da_086da227605bf568b877f13b452b7f9a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:21,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742238_1414 (size=12207) 2024-11-19T17:15:21,573 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/ddb76d7d087d4623a91d1757f72d0fff as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/ddb76d7d087d4623a91d1757f72d0fff 2024-11-19T17:15:21,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742239_1415 (size=4469) 2024-11-19T17:15:21,578 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 086da227605bf568b877f13b452b7f9a/B of 086da227605bf568b877f13b452b7f9a into ddb76d7d087d4623a91d1757f72d0fff(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:21,578 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:21,578 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., storeName=086da227605bf568b877f13b452b7f9a/B, priority=13, startTime=1732036521552; duration=0sec 2024-11-19T17:15:21,578 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:21,578 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 086da227605bf568b877f13b452b7f9a:B 2024-11-19T17:15:21,578 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:21,578 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 086da227605bf568b877f13b452b7f9a#A#compaction#353 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:21,579 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/ca8cb6de3bd24fdca019262443c5774d is 175, key is test_row_0/A:col10/1732036520669/Put/seqid=0 2024-11-19T17:15:21,580 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:21,580 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 086da227605bf568b877f13b452b7f9a/C is initiating minor compaction (all files) 2024-11-19T17:15:21,580 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 086da227605bf568b877f13b452b7f9a/C in TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:21,580 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/6f808c493e0e442e8961762552bed051, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/c67e6f9ca7344a4eb621bc734e502ed3, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/761c5122ad4b412d82d0efcb133c6903] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp, totalSize=35.3 K 2024-11-19T17:15:21,581 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f808c493e0e442e8961762552bed051, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732036516990 2024-11-19T17:15:21,581 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting c67e6f9ca7344a4eb621bc734e502ed3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732036517224 2024-11-19T17:15:21,582 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 761c5122ad4b412d82d0efcb133c6903, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732036518473 2024-11-19T17:15:21,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742240_1416 (size=31161) 2024-11-19T17:15:21,591 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 086da227605bf568b877f13b452b7f9a#C#compaction#354 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:21,591 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/8443768e83684dffbf9cbb176bc8fc80 is 50, key is test_row_0/C:col10/1732036520669/Put/seqid=0 2024-11-19T17:15:21,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742241_1417 (size=12207) 2024-11-19T17:15:21,597 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/ca8cb6de3bd24fdca019262443c5774d as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/ca8cb6de3bd24fdca019262443c5774d 2024-11-19T17:15:21,604 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/8443768e83684dffbf9cbb176bc8fc80 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/8443768e83684dffbf9cbb176bc8fc80 2024-11-19T17:15:21,606 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 086da227605bf568b877f13b452b7f9a/A of 086da227605bf568b877f13b452b7f9a into ca8cb6de3bd24fdca019262443c5774d(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:21,606 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:21,606 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., storeName=086da227605bf568b877f13b452b7f9a/A, priority=13, startTime=1732036521552; duration=0sec 2024-11-19T17:15:21,606 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:21,606 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 086da227605bf568b877f13b452b7f9a:A 2024-11-19T17:15:21,610 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 086da227605bf568b877f13b452b7f9a/C of 086da227605bf568b877f13b452b7f9a into 8443768e83684dffbf9cbb176bc8fc80(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:21,610 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:21,610 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., storeName=086da227605bf568b877f13b452b7f9a/C, priority=13, startTime=1732036521552; duration=0sec 2024-11-19T17:15:21,610 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:21,610 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 086da227605bf568b877f13b452b7f9a:C 2024-11-19T17:15:21,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-19T17:15:21,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:21,643 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 086da227605bf568b877f13b452b7f9a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T17:15:21,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=A 2024-11-19T17:15:21,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:21,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=B 2024-11-19T17:15:21,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:21,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=C 2024-11-19T17:15:21,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:21,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d87aeb9215c94fb7846b041354e297d3_086da227605bf568b877f13b452b7f9a is 50, key is test_row_0/A:col10/1732036520734/Put/seqid=0 2024-11-19T17:15:21,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T17:15:21,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742242_1418 (size=12154) 2024-11-19T17:15:21,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:21,659 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d87aeb9215c94fb7846b041354e297d3_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d87aeb9215c94fb7846b041354e297d3_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:21,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/1751ef27232f40b99991bc8ba92555ae, store: [table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:21,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/1751ef27232f40b99991bc8ba92555ae is 175, key is test_row_0/A:col10/1732036520734/Put/seqid=0 2024-11-19T17:15:21,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742243_1419 (size=30955) 2024-11-19T17:15:21,667 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/1751ef27232f40b99991bc8ba92555ae 2024-11-19T17:15:21,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/8d1915ab82b245db9bd55d6724bd7b4f is 50, key is test_row_0/B:col10/1732036520734/Put/seqid=0 2024-11-19T17:15:21,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742244_1420 (size=12001) 2024-11-19T17:15:21,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:21,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:21,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036581923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:21,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036581930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:21,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:21,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:21,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:21,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:21,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:21,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:21,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036581931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:21,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036581932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:21,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:21,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036581933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:21,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:21,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:21,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T17:15:22,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036582044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:22,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036582044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:22,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036582044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:22,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036582044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:22,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:22,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036582053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:22,082 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/8d1915ab82b245db9bd55d6724bd7b4f 2024-11-19T17:15:22,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/2371f09d2e0348f39464c34bad3f2477 is 50, key is test_row_0/C:col10/1732036520734/Put/seqid=0 2024-11-19T17:15:22,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742245_1421 (size=12001) 2024-11-19T17:15:22,106 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/2371f09d2e0348f39464c34bad3f2477 2024-11-19T17:15:22,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/1751ef27232f40b99991bc8ba92555ae as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/1751ef27232f40b99991bc8ba92555ae 2024-11-19T17:15:22,115 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/1751ef27232f40b99991bc8ba92555ae, entries=150, sequenceid=132, filesize=30.2 K 2024-11-19T17:15:22,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/8d1915ab82b245db9bd55d6724bd7b4f as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/8d1915ab82b245db9bd55d6724bd7b4f 2024-11-19T17:15:22,119 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/8d1915ab82b245db9bd55d6724bd7b4f, entries=150, sequenceid=132, filesize=11.7 K 2024-11-19T17:15:22,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/2371f09d2e0348f39464c34bad3f2477 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/2371f09d2e0348f39464c34bad3f2477 2024-11-19T17:15:22,132 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T17:15:22,132 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-19T17:15:22,135 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/2371f09d2e0348f39464c34bad3f2477, entries=150, sequenceid=132, filesize=11.7 K 2024-11-19T17:15:22,136 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 086da227605bf568b877f13b452b7f9a in 493ms, sequenceid=132, compaction requested=false 2024-11-19T17:15:22,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:22,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:22,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-19T17:15:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-19T17:15:22,139 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-19T17:15:22,140 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 801 msec 2024-11-19T17:15:22,142 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 806 msec 2024-11-19T17:15:22,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:22,259 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 086da227605bf568b877f13b452b7f9a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T17:15:22,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=A 2024-11-19T17:15:22,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:22,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=B 2024-11-19T17:15:22,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:22,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=C 2024-11-19T17:15:22,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:22,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a25f7014fb2e4ec289bf6cdc1f6a2a4c_086da227605bf568b877f13b452b7f9a is 50, key is test_row_0/A:col10/1732036521932/Put/seqid=0 2024-11-19T17:15:22,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742246_1422 (size=14794) 2024-11-19T17:15:22,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036582290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:22,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:22,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036582292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:22,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:22,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:22,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036582293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:22,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:22,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:22,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036582297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:22,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:22,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036582304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:22,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:22,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036582414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:22,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036582414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:22,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036582414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:22,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:22,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036582414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036582414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T17:15:22,474 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:15:22,475 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:15:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-19T17:15:22,477 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:15:22,478 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:15:22,478 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:15:22,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T17:15:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T17:15:22,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T17:15:22,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:22,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:22,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:22,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:22,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:22,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036582624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:22,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036582624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:22,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036582633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:22,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036582634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:22,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036582634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:22,671 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:22,674 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a25f7014fb2e4ec289bf6cdc1f6a2a4c_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a25f7014fb2e4ec289bf6cdc1f6a2a4c_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:22,675 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/63f4bee02aaa4a8688d96b8dd955c95b, store: [table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:22,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/63f4bee02aaa4a8688d96b8dd955c95b is 175, key is test_row_0/A:col10/1732036521932/Put/seqid=0 2024-11-19T17:15:22,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742247_1423 (size=39749) 2024-11-19T17:15:22,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T17:15:22,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:22,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:22,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:22,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:22,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T17:15:22,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T17:15:22,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:22,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:22,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:22,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:22,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:22,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036582943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:22,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036582943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:22,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036582954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:22,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036582954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036582954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:22,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:22,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:23,080 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/63f4bee02aaa4a8688d96b8dd955c95b 2024-11-19T17:15:23,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/1ea03b9eaa7f4eb39c6f99fc12f006fa is 50, key is test_row_0/B:col10/1732036521932/Put/seqid=0 2024-11-19T17:15:23,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T17:15:23,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:23,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:23,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:23,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:23,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:23,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742248_1424 (size=12151) 2024-11-19T17:15:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T17:15:23,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T17:15:23,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:23,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:23,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:23,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:23,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:23,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T17:15:23,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:23,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:23,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:23,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:23,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:23,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:23,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:23,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036583464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:23,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:23,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:23,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036583464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:23,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:23,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:23,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:23,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:23,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:23,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:23,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036583474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:23,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:23,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:23,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036583474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:23,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:23,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:23,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:23,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:23,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:23,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:23,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036583474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:23,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:23,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:23,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:23,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/1ea03b9eaa7f4eb39c6f99fc12f006fa 2024-11-19T17:15:23,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/491f707643184edc85e95a0b43a66a54 is 50, key is test_row_0/C:col10/1732036521932/Put/seqid=0 2024-11-19T17:15:23,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742249_1425 (size=12151) 2024-11-19T17:15:23,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T17:15:23,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:23,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:23,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:23,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:23,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:23,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T17:15:23,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T17:15:23,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:23,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:23,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:23,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:23,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:23,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:23,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T17:15:23,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:23,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:23,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:23,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:23,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:23,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/491f707643184edc85e95a0b43a66a54 2024-11-19T17:15:23,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/63f4bee02aaa4a8688d96b8dd955c95b as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/63f4bee02aaa4a8688d96b8dd955c95b 2024-11-19T17:15:23,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/63f4bee02aaa4a8688d96b8dd955c95b, entries=200, sequenceid=145, filesize=38.8 K 2024-11-19T17:15:23,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/1ea03b9eaa7f4eb39c6f99fc12f006fa as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/1ea03b9eaa7f4eb39c6f99fc12f006fa 2024-11-19T17:15:23,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/1ea03b9eaa7f4eb39c6f99fc12f006fa, entries=150, sequenceid=145, filesize=11.9 K 2024-11-19T17:15:23,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/491f707643184edc85e95a0b43a66a54 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/491f707643184edc85e95a0b43a66a54 2024-11-19T17:15:23,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/491f707643184edc85e95a0b43a66a54, entries=150, sequenceid=145, filesize=11.9 K 2024-11-19T17:15:23,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 086da227605bf568b877f13b452b7f9a in 1664ms, sequenceid=145, compaction requested=true 2024-11-19T17:15:23,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:23,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 086da227605bf568b877f13b452b7f9a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:15:23,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:23,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 086da227605bf568b877f13b452b7f9a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:15:23,924 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:23,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:23,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 086da227605bf568b877f13b452b7f9a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:15:23,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:23,924 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:23,924 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:23,925 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 086da227605bf568b877f13b452b7f9a/A is initiating minor compaction (all files) 2024-11-19T17:15:23,925 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:23,925 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 086da227605bf568b877f13b452b7f9a/B is initiating minor compaction (all files) 2024-11-19T17:15:23,925 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 086da227605bf568b877f13b452b7f9a/A in TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:23,925 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 086da227605bf568b877f13b452b7f9a/B in TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:23,925 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/ca8cb6de3bd24fdca019262443c5774d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/1751ef27232f40b99991bc8ba92555ae, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/63f4bee02aaa4a8688d96b8dd955c95b] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp, totalSize=99.5 K 2024-11-19T17:15:23,925 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/ddb76d7d087d4623a91d1757f72d0fff, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/8d1915ab82b245db9bd55d6724bd7b4f, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/1ea03b9eaa7f4eb39c6f99fc12f006fa] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp, totalSize=35.5 K 2024-11-19T17:15:23,925 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:23,925 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. files: [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/ca8cb6de3bd24fdca019262443c5774d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/1751ef27232f40b99991bc8ba92555ae, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/63f4bee02aaa4a8688d96b8dd955c95b] 2024-11-19T17:15:23,926 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting ddb76d7d087d4623a91d1757f72d0fff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732036518473 2024-11-19T17:15:23,926 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca8cb6de3bd24fdca019262443c5774d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732036518473 2024-11-19T17:15:23,926 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d1915ab82b245db9bd55d6724bd7b4f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732036520732 2024-11-19T17:15:23,926 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1751ef27232f40b99991bc8ba92555ae, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732036520732 2024-11-19T17:15:23,926 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ea03b9eaa7f4eb39c6f99fc12f006fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732036521928 2024-11-19T17:15:23,927 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63f4bee02aaa4a8688d96b8dd955c95b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732036521928 2024-11-19T17:15:23,933 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:23,934 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 086da227605bf568b877f13b452b7f9a#B#compaction#361 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:23,934 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/ddcccbf0b5da4904985fffab741f0a6d is 50, key is test_row_0/B:col10/1732036521932/Put/seqid=0 2024-11-19T17:15:23,937 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119fc6b7d258b1849de889045f4451744f0_086da227605bf568b877f13b452b7f9a store=[table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:23,939 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119fc6b7d258b1849de889045f4451744f0_086da227605bf568b877f13b452b7f9a, store=[table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:23,939 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119fc6b7d258b1849de889045f4451744f0_086da227605bf568b877f13b452b7f9a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:23,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742250_1426 (size=12459) 2024-11-19T17:15:23,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742251_1427 (size=4469) 2024-11-19T17:15:23,943 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 086da227605bf568b877f13b452b7f9a#A#compaction#362 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:23,944 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/9b6d806afe3d4195bd437b6556d1c97a is 175, key is test_row_0/A:col10/1732036521932/Put/seqid=0 2024-11-19T17:15:23,947 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/ddcccbf0b5da4904985fffab741f0a6d as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/ddcccbf0b5da4904985fffab741f0a6d 2024-11-19T17:15:23,952 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 086da227605bf568b877f13b452b7f9a/B of 086da227605bf568b877f13b452b7f9a into ddcccbf0b5da4904985fffab741f0a6d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:23,952 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:23,952 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., storeName=086da227605bf568b877f13b452b7f9a/B, priority=13, startTime=1732036523924; duration=0sec 2024-11-19T17:15:23,952 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:23,952 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 086da227605bf568b877f13b452b7f9a:B 2024-11-19T17:15:23,952 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:23,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742252_1428 (size=31413) 2024-11-19T17:15:23,953 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:23,953 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 086da227605bf568b877f13b452b7f9a/C is initiating minor compaction (all files) 2024-11-19T17:15:23,953 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 086da227605bf568b877f13b452b7f9a/C in TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:23,953 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/8443768e83684dffbf9cbb176bc8fc80, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/2371f09d2e0348f39464c34bad3f2477, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/491f707643184edc85e95a0b43a66a54] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp, totalSize=35.5 K 2024-11-19T17:15:23,954 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 8443768e83684dffbf9cbb176bc8fc80, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732036518473 2024-11-19T17:15:23,954 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 2371f09d2e0348f39464c34bad3f2477, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732036520732 2024-11-19T17:15:23,954 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 491f707643184edc85e95a0b43a66a54, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732036521928 2024-11-19T17:15:23,963 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 086da227605bf568b877f13b452b7f9a#C#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:23,964 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/22ba8c8e66c849e698b19ba4ff7d83c4 is 50, key is test_row_0/C:col10/1732036521932/Put/seqid=0 2024-11-19T17:15:23,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742253_1429 (size=12459) 2024-11-19T17:15:24,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T17:15:24,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:24,009 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 086da227605bf568b877f13b452b7f9a 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T17:15:24,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=A 2024-11-19T17:15:24,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:24,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=B 2024-11-19T17:15:24,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:24,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=C 2024-11-19T17:15:24,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:24,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111908d170ccb902444393ee8e8b4b0edda9_086da227605bf568b877f13b452b7f9a is 50, key is test_row_0/A:col10/1732036522287/Put/seqid=0 2024-11-19T17:15:24,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742254_1430 (size=12304) 2024-11-19T17:15:24,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,023 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111908d170ccb902444393ee8e8b4b0edda9_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111908d170ccb902444393ee8e8b4b0edda9_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:24,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/b78bad7f0a3b42b9b909d0d2e78b4f14, store: [table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:24,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/b78bad7f0a3b42b9b909d0d2e78b4f14 is 175, key is test_row_0/A:col10/1732036522287/Put/seqid=0 2024-11-19T17:15:24,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742255_1431 (size=31105) 2024-11-19T17:15:24,029 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=165, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/b78bad7f0a3b42b9b909d0d2e78b4f14 2024-11-19T17:15:24,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/9a5cf4e74ede420c9b08d3124f92bb2e is 50, key is test_row_0/B:col10/1732036522287/Put/seqid=0 2024-11-19T17:15:24,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742256_1432 (size=12151) 2024-11-19T17:15:24,042 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/9a5cf4e74ede420c9b08d3124f92bb2e 2024-11-19T17:15:24,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/fa7aa08a67fa41d9a3a37c3625ed71c7 is 50, key is test_row_0/C:col10/1732036522287/Put/seqid=0 2024-11-19T17:15:24,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742257_1433 (size=12151) 2024-11-19T17:15:24,356 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/9b6d806afe3d4195bd437b6556d1c97a as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/9b6d806afe3d4195bd437b6556d1c97a 2024-11-19T17:15:24,360 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 086da227605bf568b877f13b452b7f9a/A of 086da227605bf568b877f13b452b7f9a into 9b6d806afe3d4195bd437b6556d1c97a(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:24,360 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:24,360 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., storeName=086da227605bf568b877f13b452b7f9a/A, priority=13, startTime=1732036523923; duration=0sec 2024-11-19T17:15:24,360 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:24,360 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 086da227605bf568b877f13b452b7f9a:A 2024-11-19T17:15:24,371 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/22ba8c8e66c849e698b19ba4ff7d83c4 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/22ba8c8e66c849e698b19ba4ff7d83c4 2024-11-19T17:15:24,376 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 086da227605bf568b877f13b452b7f9a/C of 086da227605bf568b877f13b452b7f9a into 22ba8c8e66c849e698b19ba4ff7d83c4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:24,376 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:24,376 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., storeName=086da227605bf568b877f13b452b7f9a/C, priority=13, startTime=1732036523924; duration=0sec 2024-11-19T17:15:24,376 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:24,376 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 086da227605bf568b877f13b452b7f9a:C 2024-11-19T17:15:24,455 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/fa7aa08a67fa41d9a3a37c3625ed71c7 2024-11-19T17:15:24,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/b78bad7f0a3b42b9b909d0d2e78b4f14 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/b78bad7f0a3b42b9b909d0d2e78b4f14 2024-11-19T17:15:24,464 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/b78bad7f0a3b42b9b909d0d2e78b4f14, entries=150, sequenceid=165, filesize=30.4 K 2024-11-19T17:15:24,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/9a5cf4e74ede420c9b08d3124f92bb2e as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/9a5cf4e74ede420c9b08d3124f92bb2e 2024-11-19T17:15:24,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,468 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/9a5cf4e74ede420c9b08d3124f92bb2e, entries=150, sequenceid=165, filesize=11.9 K 2024-11-19T17:15:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/fa7aa08a67fa41d9a3a37c3625ed71c7 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/fa7aa08a67fa41d9a3a37c3625ed71c7 2024-11-19T17:15:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,475 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/fa7aa08a67fa41d9a3a37c3625ed71c7, entries=150, sequenceid=165, filesize=11.9 K 2024-11-19T17:15:24,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,476 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 086da227605bf568b877f13b452b7f9a in 467ms, sequenceid=165, compaction requested=false 2024-11-19T17:15:24,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:24,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:24,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-19T17:15:24,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-19T17:15:24,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,479 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-19T17:15:24,479 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9990 sec 2024-11-19T17:15:24,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,481 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.0050 sec 2024-11-19T17:15:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:24,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 086da227605bf568b877f13b452b7f9a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T17:15:24,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=A 2024-11-19T17:15:24,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:24,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=B 2024-11-19T17:15:24,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:24,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=C 2024-11-19T17:15:24,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T17:15:24,634 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:15:24,635 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:15:24,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-19T17:15:24,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T17:15:24,638 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:15:24,639 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:15:24,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:15:24,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ad6516faa8ac4f1c8c78933d78568696_086da227605bf568b877f13b452b7f9a is 50, key is test_row_0/A:col10/1732036524558/Put/seqid=0 2024-11-19T17:15:24,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742258_1434 (size=14794) 2024-11-19T17:15:24,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036524665, max timestamp=1732036524682Num uniques 859; 2024-11-19T17:15:24,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:15:24,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036524665, max timestamp=1732036524682Num uniques 859; 2024-11-19T17:15:24,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036524665, max timestamp=1732036524682Num uniques 859; 2024-11-19T17:15:24,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:15:24,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:15:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T17:15:24,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036584726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:24,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:24,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:24,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:24,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:24,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036584726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036584727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:24,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:24,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:24,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:24,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036584728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036584728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:24,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:24,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:24,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:24,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:24,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:24,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:24,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:24,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:24,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:24,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:24,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T17:15:24,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:24,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:24,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:24,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:24,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:24,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036584854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:24,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:24,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:24,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036584854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:24,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:24,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:24,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:24,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:24,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036584854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:24,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:24,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:24,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:24,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036584855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:24,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:24,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:24,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:24,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036584855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:24,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:24,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:24,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:24,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T17:15:24,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:24,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:24,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:24,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:24,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T17:15:25,055 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:25,058 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ad6516faa8ac4f1c8c78933d78568696_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ad6516faa8ac4f1c8c78933d78568696_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:25,059 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/7f7accba6d714a648f5e5fb2dbfa8dda, store: [table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:25,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/7f7accba6d714a648f5e5fb2dbfa8dda is 175, key is test_row_0/A:col10/1732036524558/Put/seqid=0 2024-11-19T17:15:25,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742259_1435 (size=39745) 2024-11-19T17:15:25,065 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=177, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/7f7accba6d714a648f5e5fb2dbfa8dda 2024-11-19T17:15:25,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:25,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036585063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:25,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:25,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:25,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:25,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036585064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:25,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:25,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:25,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:25,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:25,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:25,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036585064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:25,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:25,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:25,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:25,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:25,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036585064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:25,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:25,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:25,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/3c6a5ca5b433441181b7aa17be9ebc29 is 50, key is test_row_0/B:col10/1732036524558/Put/seqid=0 2024-11-19T17:15:25,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:25,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036585064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:25,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:25,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:25,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:25,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:25,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742260_1436 (size=9757) 2024-11-19T17:15:25,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T17:15:25,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:25,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:25,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:25,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:25,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:25,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:25,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T17:15:25,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:25,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:25,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:25,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:25,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:25,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:25,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T17:15:25,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:25,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036585384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:25,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:25,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036585384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:25,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:25,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036585384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:25,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:25,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036585384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:25,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:25,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036585384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:25,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T17:15:25,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:25,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:25,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:25,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:25,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:25,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:25,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/3c6a5ca5b433441181b7aa17be9ebc29 2024-11-19T17:15:25,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/e3ae8aa4ca12481cae59a074efa1c336 is 50, key is test_row_0/C:col10/1732036524558/Put/seqid=0 2024-11-19T17:15:25,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742261_1437 (size=9757) 2024-11-19T17:15:25,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T17:15:25,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:25,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:25,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:25,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:25,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:25,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T17:15:25,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:25,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:25,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:25,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:25,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:25,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:25,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T17:15:25,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T17:15:25,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:25,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:25,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:25,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:25,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:25,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:25,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/e3ae8aa4ca12481cae59a074efa1c336 2024-11-19T17:15:25,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/7f7accba6d714a648f5e5fb2dbfa8dda as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/7f7accba6d714a648f5e5fb2dbfa8dda 2024-11-19T17:15:25,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/7f7accba6d714a648f5e5fb2dbfa8dda, entries=200, sequenceid=177, filesize=38.8 K 2024-11-19T17:15:25,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/3c6a5ca5b433441181b7aa17be9ebc29 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/3c6a5ca5b433441181b7aa17be9ebc29 2024-11-19T17:15:25,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:25,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036585903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:25,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:25,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:25,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:25,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036585904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:25,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:25,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:25,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036585904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:25,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:25,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:25,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:25,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:25,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:25,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:25,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:25,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036585904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:25,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:25,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:25,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:25,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:25,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036585904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:25,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:25,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:25,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:25,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/3c6a5ca5b433441181b7aa17be9ebc29, entries=100, sequenceid=177, filesize=9.5 K 2024-11-19T17:15:25,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/e3ae8aa4ca12481cae59a074efa1c336 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/e3ae8aa4ca12481cae59a074efa1c336 2024-11-19T17:15:25,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/e3ae8aa4ca12481cae59a074efa1c336, entries=100, sequenceid=177, filesize=9.5 K 2024-11-19T17:15:25,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 086da227605bf568b877f13b452b7f9a in 1286ms, sequenceid=177, compaction requested=true 2024-11-19T17:15:25,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:25,918 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:25,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 086da227605bf568b877f13b452b7f9a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:15:25,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:25,918 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:25,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 086da227605bf568b877f13b452b7f9a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:15:25,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:25,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 086da227605bf568b877f13b452b7f9a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:15:25,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:25,919 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:25,919 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:25,919 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 086da227605bf568b877f13b452b7f9a/A is initiating minor compaction (all files) 2024-11-19T17:15:25,919 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 086da227605bf568b877f13b452b7f9a/B is initiating minor compaction (all files) 2024-11-19T17:15:25,919 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 086da227605bf568b877f13b452b7f9a/A in TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:25,919 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 086da227605bf568b877f13b452b7f9a/B in TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:25,919 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/ddcccbf0b5da4904985fffab741f0a6d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/9a5cf4e74ede420c9b08d3124f92bb2e, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/3c6a5ca5b433441181b7aa17be9ebc29] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp, totalSize=33.6 K 2024-11-19T17:15:25,919 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/9b6d806afe3d4195bd437b6556d1c97a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/b78bad7f0a3b42b9b909d0d2e78b4f14, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/7f7accba6d714a648f5e5fb2dbfa8dda] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp, totalSize=99.9 K 2024-11-19T17:15:25,919 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:25,920 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. files: [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/9b6d806afe3d4195bd437b6556d1c97a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/b78bad7f0a3b42b9b909d0d2e78b4f14, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/7f7accba6d714a648f5e5fb2dbfa8dda] 2024-11-19T17:15:25,920 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting ddcccbf0b5da4904985fffab741f0a6d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732036521928 2024-11-19T17:15:25,920 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b6d806afe3d4195bd437b6556d1c97a, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732036521928 2024-11-19T17:15:25,920 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting b78bad7f0a3b42b9b909d0d2e78b4f14, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732036522287 2024-11-19T17:15:25,920 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a5cf4e74ede420c9b08d3124f92bb2e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732036522287 2024-11-19T17:15:25,920 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c6a5ca5b433441181b7aa17be9ebc29, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732036524558 2024-11-19T17:15:25,920 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f7accba6d714a648f5e5fb2dbfa8dda, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732036524558 2024-11-19T17:15:25,926 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:25,928 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 086da227605bf568b877f13b452b7f9a#B#compaction#371 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:25,928 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/05a8f002ff644661b11a9d974fbe8039 is 50, key is test_row_0/B:col10/1732036524558/Put/seqid=0 2024-11-19T17:15:25,929 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111936864e09ea944b7386d6815e654c706c_086da227605bf568b877f13b452b7f9a store=[table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:25,931 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111936864e09ea944b7386d6815e654c706c_086da227605bf568b877f13b452b7f9a, store=[table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:25,931 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111936864e09ea944b7386d6815e654c706c_086da227605bf568b877f13b452b7f9a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:25,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742262_1438 (size=12561) 2024-11-19T17:15:25,941 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/05a8f002ff644661b11a9d974fbe8039 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/05a8f002ff644661b11a9d974fbe8039 2024-11-19T17:15:25,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742263_1439 (size=4469) 2024-11-19T17:15:25,948 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 086da227605bf568b877f13b452b7f9a/B of 086da227605bf568b877f13b452b7f9a into 05a8f002ff644661b11a9d974fbe8039(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:25,948 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:25,948 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., storeName=086da227605bf568b877f13b452b7f9a/B, priority=13, startTime=1732036525918; duration=0sec 2024-11-19T17:15:25,949 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:25,949 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 086da227605bf568b877f13b452b7f9a:B 2024-11-19T17:15:25,949 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:25,950 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:25,950 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 086da227605bf568b877f13b452b7f9a/C is initiating minor compaction (all files) 2024-11-19T17:15:25,950 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 086da227605bf568b877f13b452b7f9a/C in TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:25,950 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/22ba8c8e66c849e698b19ba4ff7d83c4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/fa7aa08a67fa41d9a3a37c3625ed71c7, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/e3ae8aa4ca12481cae59a074efa1c336] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp, totalSize=33.6 K 2024-11-19T17:15:25,950 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 22ba8c8e66c849e698b19ba4ff7d83c4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732036521928 2024-11-19T17:15:25,950 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting fa7aa08a67fa41d9a3a37c3625ed71c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732036522287 2024-11-19T17:15:25,951 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting e3ae8aa4ca12481cae59a074efa1c336, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732036524558 2024-11-19T17:15:25,957 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 086da227605bf568b877f13b452b7f9a#C#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:25,957 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/ccc6364e42154eb086e4a656de1f8944 is 50, key is test_row_0/C:col10/1732036524558/Put/seqid=0 2024-11-19T17:15:25,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742264_1440 (size=12561) 2024-11-19T17:15:25,966 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/ccc6364e42154eb086e4a656de1f8944 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/ccc6364e42154eb086e4a656de1f8944 2024-11-19T17:15:25,969 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 086da227605bf568b877f13b452b7f9a/C of 086da227605bf568b877f13b452b7f9a into ccc6364e42154eb086e4a656de1f8944(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:25,969 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:25,969 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., storeName=086da227605bf568b877f13b452b7f9a/C, priority=13, startTime=1732036525919; duration=0sec 2024-11-19T17:15:25,969 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:25,969 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 086da227605bf568b877f13b452b7f9a:C 2024-11-19T17:15:26,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T17:15:26,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:26,017 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 086da227605bf568b877f13b452b7f9a 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-19T17:15:26,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=A 2024-11-19T17:15:26,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:26,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=B 2024-11-19T17:15:26,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:26,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=C 2024-11-19T17:15:26,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:26,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119345b2c158b944f499e36c5ab2416ca9a_086da227605bf568b877f13b452b7f9a is 50, key is test_row_0/A:col10/1732036524722/Put/seqid=0 2024-11-19T17:15:26,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742265_1441 (size=12304) 2024-11-19T17:15:26,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,037 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119345b2c158b944f499e36c5ab2416ca9a_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119345b2c158b944f499e36c5ab2416ca9a_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:26,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/69d92541c11d432f897ad4f9cdc28af9, store: [table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:26,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/69d92541c11d432f897ad4f9cdc28af9 is 175, key is test_row_0/A:col10/1732036524722/Put/seqid=0 2024-11-19T17:15:26,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742266_1442 (size=31105) 2024-11-19T17:15:26,344 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 086da227605bf568b877f13b452b7f9a#A#compaction#370 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:26,344 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/44212c4215bc478095d0d3da90270482 is 175, key is test_row_0/A:col10/1732036524558/Put/seqid=0 2024-11-19T17:15:26,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742267_1443 (size=31622) 2024-11-19T17:15:26,449 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=215, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/69d92541c11d432f897ad4f9cdc28af9 2024-11-19T17:15:26,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/c01ecc84631e49a897dca99b174a7a2e is 50, key is test_row_0/B:col10/1732036524722/Put/seqid=0 2024-11-19T17:15:26,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742268_1444 (size=12151) 2024-11-19T17:15:26,752 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/44212c4215bc478095d0d3da90270482 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/44212c4215bc478095d0d3da90270482 2024-11-19T17:15:26,756 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 086da227605bf568b877f13b452b7f9a/A of 086da227605bf568b877f13b452b7f9a into 44212c4215bc478095d0d3da90270482(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:26,756 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:26,756 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., storeName=086da227605bf568b877f13b452b7f9a/A, priority=13, startTime=1732036525918; duration=0sec 2024-11-19T17:15:26,756 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:26,756 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 086da227605bf568b877f13b452b7f9a:A 2024-11-19T17:15:26,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T17:15:26,863 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/c01ecc84631e49a897dca99b174a7a2e 2024-11-19T17:15:26,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/40f91a2a896a473597f63bea3134495a is 50, key is test_row_0/C:col10/1732036524722/Put/seqid=0 2024-11-19T17:15:26,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742269_1445 (size=12151) 2024-11-19T17:15:26,875 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/40f91a2a896a473597f63bea3134495a 2024-11-19T17:15:26,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/69d92541c11d432f897ad4f9cdc28af9 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/69d92541c11d432f897ad4f9cdc28af9 2024-11-19T17:15:26,883 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/69d92541c11d432f897ad4f9cdc28af9, entries=150, sequenceid=215, filesize=30.4 K 2024-11-19T17:15:26,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/c01ecc84631e49a897dca99b174a7a2e as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/c01ecc84631e49a897dca99b174a7a2e 2024-11-19T17:15:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,888 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/c01ecc84631e49a897dca99b174a7a2e, entries=150, sequenceid=215, filesize=11.9 K 2024-11-19T17:15:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/40f91a2a896a473597f63bea3134495a as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/40f91a2a896a473597f63bea3134495a 2024-11-19T17:15:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,893 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/40f91a2a896a473597f63bea3134495a, entries=150, sequenceid=215, filesize=11.9 K 2024-11-19T17:15:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,894 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=0 B/0 for 086da227605bf568b877f13b452b7f9a in 877ms, sequenceid=215, compaction requested=false 2024-11-19T17:15:26,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:26,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-19T17:15:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-19T17:15:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-19T17:15:26,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2560 sec 2024-11-19T17:15:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.2640 sec 2024-11-19T17:15:26,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 086da227605bf568b877f13b452b7f9a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T17:15:27,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=A 2024-11-19T17:15:27,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=B 2024-11-19T17:15:27,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:27,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=C 2024-11-19T17:15:27,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b2cb8f881dde440b90911487e7d826a5_086da227605bf568b877f13b452b7f9a is 50, key is test_row_0/A:col10/1732036527004/Put/seqid=0 2024-11-19T17:15:27,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742270_1446 (size=22268) 2024-11-19T17:15:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732036527057, max timestamp=1732036527085Num uniques 859; 2024-11-19T17:15:27,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:15:27,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732036527057, max timestamp=1732036527085Num uniques 859; 2024-11-19T17:15:27,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:15:27,091 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036527057, max timestamp=1732036527085Num uniques 859; 2024-11-19T17:15:27,091 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:15:27,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036587128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:27,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036587128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:27,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036587129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:27,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:27,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036587131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:27,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:27,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:27,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036587132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:27,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:27,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:27,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:27,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036587254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:27,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:27,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036587254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:27,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:27,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:27,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036587254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:27,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:27,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:27,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036587254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:27,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:27,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:27,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036587254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:27,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:27,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:27,429 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:27,433 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b2cb8f881dde440b90911487e7d826a5_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b2cb8f881dde440b90911487e7d826a5_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:27,434 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/7e1a7b8eb331452baf7b606829f5c886, store: [table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:27,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/7e1a7b8eb331452baf7b606829f5c886 is 175, key is test_row_0/A:col10/1732036527004/Put/seqid=0 2024-11-19T17:15:27,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742271_1447 (size=65673) 2024-11-19T17:15:27,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036587473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:27,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:27,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036587473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:27,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:27,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:27,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036587473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:27,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:27,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:27,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036587473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:27,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:27,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:27,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036587473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:27,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:27,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:27,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036587784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:27,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:27,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036587784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:27,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:27,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:27,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036587784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:27,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036587784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:27,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:27,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036587793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:27,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:27,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:27,841 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/7e1a7b8eb331452baf7b606829f5c886 2024-11-19T17:15:27,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/7294b2c8bd6b47bdb6067c858c9ce657 is 50, key is test_row_0/B:col10/1732036527004/Put/seqid=0 2024-11-19T17:15:27,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742272_1448 (size=12151) 2024-11-19T17:15:27,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/7294b2c8bd6b47bdb6067c858c9ce657 2024-11-19T17:15:27,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/d6ac24ee8e334e7ea0b5d002992fce41 is 50, key is test_row_0/C:col10/1732036527004/Put/seqid=0 2024-11-19T17:15:27,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742273_1449 (size=12151) 2024-11-19T17:15:27,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/d6ac24ee8e334e7ea0b5d002992fce41 2024-11-19T17:15:27,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/7e1a7b8eb331452baf7b606829f5c886 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/7e1a7b8eb331452baf7b606829f5c886 2024-11-19T17:15:27,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/7e1a7b8eb331452baf7b606829f5c886, entries=350, sequenceid=226, filesize=64.1 K 2024-11-19T17:15:27,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/7294b2c8bd6b47bdb6067c858c9ce657 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/7294b2c8bd6b47bdb6067c858c9ce657 2024-11-19T17:15:27,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/7294b2c8bd6b47bdb6067c858c9ce657, entries=150, sequenceid=226, filesize=11.9 K 2024-11-19T17:15:27,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/d6ac24ee8e334e7ea0b5d002992fce41 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/d6ac24ee8e334e7ea0b5d002992fce41 2024-11-19T17:15:27,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/d6ac24ee8e334e7ea0b5d002992fce41, entries=150, sequenceid=226, filesize=11.9 K 2024-11-19T17:15:27,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 086da227605bf568b877f13b452b7f9a in 873ms, sequenceid=226, compaction requested=true 2024-11-19T17:15:27,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:27,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 086da227605bf568b877f13b452b7f9a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:15:27,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:27,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 086da227605bf568b877f13b452b7f9a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:15:27,878 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:27,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:27,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 086da227605bf568b877f13b452b7f9a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:15:27,878 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:27,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:27,879 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128400 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:27,879 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:27,879 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 086da227605bf568b877f13b452b7f9a/B is initiating minor compaction (all files) 2024-11-19T17:15:27,879 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 086da227605bf568b877f13b452b7f9a/A is initiating minor compaction (all files) 2024-11-19T17:15:27,879 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 086da227605bf568b877f13b452b7f9a/A in TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:27,879 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 086da227605bf568b877f13b452b7f9a/B in TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:27,879 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/05a8f002ff644661b11a9d974fbe8039, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/c01ecc84631e49a897dca99b174a7a2e, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/7294b2c8bd6b47bdb6067c858c9ce657] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp, totalSize=36.0 K 2024-11-19T17:15:27,879 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/44212c4215bc478095d0d3da90270482, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/69d92541c11d432f897ad4f9cdc28af9, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/7e1a7b8eb331452baf7b606829f5c886] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp, totalSize=125.4 K 2024-11-19T17:15:27,879 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:27,879 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. files: [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/44212c4215bc478095d0d3da90270482, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/69d92541c11d432f897ad4f9cdc28af9, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/7e1a7b8eb331452baf7b606829f5c886] 2024-11-19T17:15:27,879 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44212c4215bc478095d0d3da90270482, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732036522295 2024-11-19T17:15:27,879 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 05a8f002ff644661b11a9d974fbe8039, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732036522295 2024-11-19T17:15:27,880 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting c01ecc84631e49a897dca99b174a7a2e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732036524722 2024-11-19T17:15:27,880 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69d92541c11d432f897ad4f9cdc28af9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732036524722 2024-11-19T17:15:27,880 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 7294b2c8bd6b47bdb6067c858c9ce657, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732036526978 2024-11-19T17:15:27,880 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e1a7b8eb331452baf7b606829f5c886, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732036526976 2024-11-19T17:15:27,888 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 086da227605bf568b877f13b452b7f9a#B#compaction#379 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:27,888 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/976eb33c834149b9a0ab268a415f1cb9 is 50, key is test_row_0/B:col10/1732036527004/Put/seqid=0 2024-11-19T17:15:27,889 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:27,894 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411194e849679a3c34cf7a9e7fd6d11e70dfe_086da227605bf568b877f13b452b7f9a store=[table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:27,896 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411194e849679a3c34cf7a9e7fd6d11e70dfe_086da227605bf568b877f13b452b7f9a, store=[table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:27,896 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194e849679a3c34cf7a9e7fd6d11e70dfe_086da227605bf568b877f13b452b7f9a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:27,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742274_1450 (size=12663) 2024-11-19T17:15:27,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742275_1451 (size=4469) 2024-11-19T17:15:27,906 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/976eb33c834149b9a0ab268a415f1cb9 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/976eb33c834149b9a0ab268a415f1cb9 2024-11-19T17:15:27,907 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 086da227605bf568b877f13b452b7f9a#A#compaction#380 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:27,908 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/8da0e1421e574f9caed7b49433baa2fe is 175, key is test_row_0/A:col10/1732036527004/Put/seqid=0 2024-11-19T17:15:27,913 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 086da227605bf568b877f13b452b7f9a/B of 086da227605bf568b877f13b452b7f9a into 976eb33c834149b9a0ab268a415f1cb9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:27,913 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:27,913 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., storeName=086da227605bf568b877f13b452b7f9a/B, priority=13, startTime=1732036527878; duration=0sec 2024-11-19T17:15:27,913 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:27,913 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 086da227605bf568b877f13b452b7f9a:B 2024-11-19T17:15:27,913 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:27,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742276_1452 (size=31617) 2024-11-19T17:15:27,916 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:27,916 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 086da227605bf568b877f13b452b7f9a/C is initiating minor compaction (all files) 2024-11-19T17:15:27,916 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 086da227605bf568b877f13b452b7f9a/C in TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:27,916 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/ccc6364e42154eb086e4a656de1f8944, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/40f91a2a896a473597f63bea3134495a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/d6ac24ee8e334e7ea0b5d002992fce41] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp, totalSize=36.0 K 2024-11-19T17:15:27,917 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting ccc6364e42154eb086e4a656de1f8944, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732036522295 2024-11-19T17:15:27,920 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 40f91a2a896a473597f63bea3134495a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732036524722 2024-11-19T17:15:27,920 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting d6ac24ee8e334e7ea0b5d002992fce41, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732036526978 2024-11-19T17:15:27,922 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/8da0e1421e574f9caed7b49433baa2fe as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/8da0e1421e574f9caed7b49433baa2fe 2024-11-19T17:15:27,927 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 086da227605bf568b877f13b452b7f9a/A of 086da227605bf568b877f13b452b7f9a into 8da0e1421e574f9caed7b49433baa2fe(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:27,927 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:27,927 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., storeName=086da227605bf568b877f13b452b7f9a/A, priority=13, startTime=1732036527878; duration=0sec 2024-11-19T17:15:27,927 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:27,927 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 086da227605bf568b877f13b452b7f9a:A 2024-11-19T17:15:27,927 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 086da227605bf568b877f13b452b7f9a#C#compaction#381 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:27,928 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/75917db73f224eea82256fe9dd21587d is 50, key is test_row_0/C:col10/1732036527004/Put/seqid=0 2024-11-19T17:15:27,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742277_1453 (size=12663) 2024-11-19T17:15:27,941 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/75917db73f224eea82256fe9dd21587d as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/75917db73f224eea82256fe9dd21587d 2024-11-19T17:15:27,945 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 086da227605bf568b877f13b452b7f9a/C of 086da227605bf568b877f13b452b7f9a into 75917db73f224eea82256fe9dd21587d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:27,945 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:27,945 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., storeName=086da227605bf568b877f13b452b7f9a/C, priority=13, startTime=1732036527878; duration=0sec 2024-11-19T17:15:27,945 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:27,945 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 086da227605bf568b877f13b452b7f9a:C 2024-11-19T17:15:28,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:28,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 086da227605bf568b877f13b452b7f9a 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-19T17:15:28,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=A 2024-11-19T17:15:28,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:28,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=B 2024-11-19T17:15:28,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:28,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=C 2024-11-19T17:15:28,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:28,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198887a8dfe1144b179a63311537fe3106_086da227605bf568b877f13b452b7f9a is 50, key is test_row_0/A:col10/1732036528309/Put/seqid=0 2024-11-19T17:15:28,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742278_1454 (size=14944) 2024-11-19T17:15:28,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:28,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036588316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:28,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:28,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:28,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036588327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:28,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:28,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036588327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:28,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:28,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:28,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:28,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036588328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:28,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:28,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:28,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036588330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:28,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:28,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:28,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036588434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:28,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:28,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:28,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036588434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:28,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:28,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036588434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:28,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036588443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:28,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036588443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:28,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036588654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:28,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:28,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036588654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:28,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:28,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036588654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:28,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:28,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036588654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:28,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036588654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:28,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:28,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:28,724 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:28,727 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198887a8dfe1144b179a63311537fe3106_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198887a8dfe1144b179a63311537fe3106_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:28,728 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/018d8d52807e44ffa56725a24796c8b0, store: [table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:28,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/018d8d52807e44ffa56725a24796c8b0 is 175, key is test_row_0/A:col10/1732036528309/Put/seqid=0 2024-11-19T17:15:28,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742279_1455 (size=39899) 2024-11-19T17:15:28,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T17:15:28,814 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:15:28,814 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:15:28,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-19T17:15:28,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T17:15:28,816 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:15:28,817 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:15:28,817 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:15:28,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T17:15:28,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T17:15:28,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:28,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:28,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:28,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:28,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:28,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:28,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:28,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036588975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:28,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:28,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036588976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:28,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:28,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036588976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:28,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:28,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:28,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:28,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036588976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:28,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:28,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:28,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036588977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:28,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:28,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:29,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T17:15:29,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:29,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:29,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:29,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:29,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:29,132 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=266, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/018d8d52807e44ffa56725a24796c8b0 2024-11-19T17:15:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T17:15:29,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/93129d8219f8478196ced2c66af3a038 is 50, key is test_row_0/B:col10/1732036528309/Put/seqid=0 2024-11-19T17:15:29,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742280_1456 (size=12251) 2024-11-19T17:15:29,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/93129d8219f8478196ced2c66af3a038 2024-11-19T17:15:29,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/a329770dcdfe4bd48ae54b94944b9a62 is 50, key is test_row_0/C:col10/1732036528309/Put/seqid=0 2024-11-19T17:15:29,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742281_1457 (size=12251) 2024-11-19T17:15:29,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/a329770dcdfe4bd48ae54b94944b9a62 2024-11-19T17:15:29,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/018d8d52807e44ffa56725a24796c8b0 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/018d8d52807e44ffa56725a24796c8b0 2024-11-19T17:15:29,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/018d8d52807e44ffa56725a24796c8b0, entries=200, sequenceid=266, filesize=39.0 K 2024-11-19T17:15:29,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/93129d8219f8478196ced2c66af3a038 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/93129d8219f8478196ced2c66af3a038 2024-11-19T17:15:29,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/93129d8219f8478196ced2c66af3a038, entries=150, sequenceid=266, filesize=12.0 K 2024-11-19T17:15:29,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/a329770dcdfe4bd48ae54b94944b9a62 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/a329770dcdfe4bd48ae54b94944b9a62 2024-11-19T17:15:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/a329770dcdfe4bd48ae54b94944b9a62, entries=150, sequenceid=266, filesize=12.0 K 2024-11-19T17:15:29,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=46.96 KB/48090 for 086da227605bf568b877f13b452b7f9a in 979ms, sequenceid=266, compaction requested=false 2024-11-19T17:15:29,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:29,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T17:15:29,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,292 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 086da227605bf568b877f13b452b7f9a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T17:15:29,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=A 2024-11-19T17:15:29,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:29,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=B 2024-11-19T17:15:29,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:29,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=C 2024-11-19T17:15:29,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119976c8d5143a844f8814805b6b04b6c79_086da227605bf568b877f13b452b7f9a is 50, key is test_row_1/A:col10/1732036528327/Put/seqid=0 2024-11-19T17:15:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742282_1458 (size=9914) 2024-11-19T17:15:29,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,335 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119976c8d5143a844f8814805b6b04b6c79_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119976c8d5143a844f8814805b6b04b6c79_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/bf185c309ff647c8ae42b83989b3e837, store: [table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/bf185c309ff647c8ae42b83989b3e837 is 175, key is test_row_1/A:col10/1732036528327/Put/seqid=0 2024-11-19T17:15:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742283_1459 (size=22561) 2024-11-19T17:15:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T17:15:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:29,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:29,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036529586, max timestamp=1732036529612Num uniques 859; 2024-11-19T17:15:29,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036529586, max timestamp=1732036529612Num uniques 859; 2024-11-19T17:15:29,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:15:29,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:15:29,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036529586, max timestamp=1732036529612Num uniques 859; 2024-11-19T17:15:29,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:15:29,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:29,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036589647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:29,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:29,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:29,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036589648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:29,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:29,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:29,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:29,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:29,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:29,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:29,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036589650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:29,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:29,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036589653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:29,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:29,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:29,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:29,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:29,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:29,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:29,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:29,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036589661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:29,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:29,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:29,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:29,767 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/bf185c309ff647c8ae42b83989b3e837 2024-11-19T17:15:29,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/8df7d19114944f05aa0d06f07aec1f8e is 50, key is test_row_1/B:col10/1732036528327/Put/seqid=0 2024-11-19T17:15:29,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742284_1460 (size=9857) 2024-11-19T17:15:29,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036589777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:29,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:29,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036589777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:29,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:29,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036589777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:29,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:29,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036589777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:29,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:29,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:29,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:29,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:29,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036589783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:29,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:29,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:29,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:29,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T17:15:30,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:30,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036589994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:30,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:30,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036589994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:30,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:30,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:30,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:30,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036589994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:30,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:30,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:30,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036589994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:30,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:30,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:30,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036589994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:30,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:30,183 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/8df7d19114944f05aa0d06f07aec1f8e 2024-11-19T17:15:30,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/9402284eb91d4b4a897c8528b4ff673c is 50, key is test_row_1/C:col10/1732036528327/Put/seqid=0 2024-11-19T17:15:30,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742285_1461 (size=9857) 2024-11-19T17:15:30,199 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/9402284eb91d4b4a897c8528b4ff673c 2024-11-19T17:15:30,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/bf185c309ff647c8ae42b83989b3e837 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/bf185c309ff647c8ae42b83989b3e837 2024-11-19T17:15:30,206 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/bf185c309ff647c8ae42b83989b3e837, entries=100, sequenceid=276, filesize=22.0 K 2024-11-19T17:15:30,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/8df7d19114944f05aa0d06f07aec1f8e as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/8df7d19114944f05aa0d06f07aec1f8e 2024-11-19T17:15:30,209 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/8df7d19114944f05aa0d06f07aec1f8e, entries=100, sequenceid=276, filesize=9.6 K 2024-11-19T17:15:30,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/9402284eb91d4b4a897c8528b4ff673c as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/9402284eb91d4b4a897c8528b4ff673c 2024-11-19T17:15:30,213 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/9402284eb91d4b4a897c8528b4ff673c, entries=100, sequenceid=276, filesize=9.6 K 2024-11-19T17:15:30,214 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 086da227605bf568b877f13b452b7f9a in 922ms, sequenceid=276, compaction requested=true 2024-11-19T17:15:30,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:30,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:30,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-19T17:15:30,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-19T17:15:30,217 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-19T17:15:30,217 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3980 sec 2024-11-19T17:15:30,218 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.4020 sec 2024-11-19T17:15:30,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:30,323 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 086da227605bf568b877f13b452b7f9a 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-19T17:15:30,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=A 2024-11-19T17:15:30,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:30,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=B 2024-11-19T17:15:30,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:30,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=C 2024-11-19T17:15:30,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:30,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197269be4816ab40ea99268c932c897b9b_086da227605bf568b877f13b452b7f9a is 50, key is test_row_0/A:col10/1732036530321/Put/seqid=0 2024-11-19T17:15:30,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742286_1462 (size=14994) 2024-11-19T17:15:30,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:30,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:30,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036590327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:30,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036590328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:30,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:30,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:30,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:30,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036590343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:30,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:30,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:30,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036590344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:30,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:30,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036590344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:30,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:30,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:30,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:30,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036590453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:30,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:30,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036590453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:30,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:30,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:30,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036590453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:30,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:30,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:30,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:30,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036590464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:30,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:30,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036590464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:30,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:30,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:30,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036590673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:30,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036590674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:30,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:30,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036590674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:30,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:30,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036590684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:30,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:30,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036590684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:30,735 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:30,738 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197269be4816ab40ea99268c932c897b9b_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197269be4816ab40ea99268c932c897b9b_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:30,739 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/ad1d7d8c3b5147ed9187933ed37aadc1, store: [table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:30,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/ad1d7d8c3b5147ed9187933ed37aadc1 is 175, key is test_row_0/A:col10/1732036530321/Put/seqid=0 2024-11-19T17:15:30,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742287_1463 (size=39949) 2024-11-19T17:15:30,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T17:15:30,963 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:15:30,964 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:15:30,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-19T17:15:30,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T17:15:30,966 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:15:30,966 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:15:30,967 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:15:31,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:31,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036590993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:31,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:31,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:31,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:31,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:31,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036590994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:31,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:31,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:31,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:31,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:31,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036590994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:31,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:31,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:31,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:31,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:31,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036590994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:31,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:31,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:31,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:31,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:31,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036590994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:31,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:31,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:31,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T17:15:31,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T17:15:31,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:31,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:31,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:31,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:31,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:31,143 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/ad1d7d8c3b5147ed9187933ed37aadc1 2024-11-19T17:15:31,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/3f588ec74f334bab992909bc34527cea is 50, key is test_row_0/B:col10/1732036530321/Put/seqid=0 2024-11-19T17:15:31,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742288_1464 (size=12301) 2024-11-19T17:15:31,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T17:15:31,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:31,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:31,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:31,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:31,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:31,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:31,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T17:15:31,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T17:15:31,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:31,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:31,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:31,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:31,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:31,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036591514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:31,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:31,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:31,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036591514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:31,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:31,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:31,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:31,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:31,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036591514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:31,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:31,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:31,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036591514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:31,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:31,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:31,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:31,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:31,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036591514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:31,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:31,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:31,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:31,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/3f588ec74f334bab992909bc34527cea 2024-11-19T17:15:31,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/8111d3a2913e4ec68b8751732135b856 is 50, key is test_row_0/C:col10/1732036530321/Put/seqid=0 2024-11-19T17:15:31,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742289_1465 (size=12301) 2024-11-19T17:15:31,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T17:15:31,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:31,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:31,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:31,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:31,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T17:15:31,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T17:15:31,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:31,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:31,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:31,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:31,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:31,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:31,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T17:15:31,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:31,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:31,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:31,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:31,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:31,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:31,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/8111d3a2913e4ec68b8751732135b856 2024-11-19T17:15:31,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/ad1d7d8c3b5147ed9187933ed37aadc1 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/ad1d7d8c3b5147ed9187933ed37aadc1 2024-11-19T17:15:31,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/ad1d7d8c3b5147ed9187933ed37aadc1, entries=200, sequenceid=313, filesize=39.0 K 2024-11-19T17:15:31,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/3f588ec74f334bab992909bc34527cea as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/3f588ec74f334bab992909bc34527cea 2024-11-19T17:15:31,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/3f588ec74f334bab992909bc34527cea, entries=150, sequenceid=313, filesize=12.0 K 2024-11-19T17:15:31,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/8111d3a2913e4ec68b8751732135b856 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/8111d3a2913e4ec68b8751732135b856 2024-11-19T17:15:31,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/8111d3a2913e4ec68b8751732135b856, entries=150, sequenceid=313, filesize=12.0 K 2024-11-19T17:15:31,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=46.96 KB/48090 for 086da227605bf568b877f13b452b7f9a in 1660ms, sequenceid=313, compaction requested=true 2024-11-19T17:15:31,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:31,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 086da227605bf568b877f13b452b7f9a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:15:31,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:31,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 086da227605bf568b877f13b452b7f9a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:15:31,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:31,982 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:15:31,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 086da227605bf568b877f13b452b7f9a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:15:31,982 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:15:31,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:31,983 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134026 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:15:31,983 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47072 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:15:31,983 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 086da227605bf568b877f13b452b7f9a/A is initiating minor compaction (all files) 2024-11-19T17:15:31,983 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 086da227605bf568b877f13b452b7f9a/B is initiating minor compaction (all files) 2024-11-19T17:15:31,983 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 086da227605bf568b877f13b452b7f9a/A in TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:31,983 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 086da227605bf568b877f13b452b7f9a/B in TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:31,984 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/8da0e1421e574f9caed7b49433baa2fe, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/018d8d52807e44ffa56725a24796c8b0, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/bf185c309ff647c8ae42b83989b3e837, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/ad1d7d8c3b5147ed9187933ed37aadc1] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp, totalSize=130.9 K 2024-11-19T17:15:31,984 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/976eb33c834149b9a0ab268a415f1cb9, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/93129d8219f8478196ced2c66af3a038, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/8df7d19114944f05aa0d06f07aec1f8e, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/3f588ec74f334bab992909bc34527cea] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp, totalSize=46.0 K 2024-11-19T17:15:31,984 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:31,984 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. files: [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/8da0e1421e574f9caed7b49433baa2fe, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/018d8d52807e44ffa56725a24796c8b0, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/bf185c309ff647c8ae42b83989b3e837, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/ad1d7d8c3b5147ed9187933ed37aadc1] 2024-11-19T17:15:31,984 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 976eb33c834149b9a0ab268a415f1cb9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732036526978 2024-11-19T17:15:31,984 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8da0e1421e574f9caed7b49433baa2fe, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732036526978 2024-11-19T17:15:31,984 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 93129d8219f8478196ced2c66af3a038, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732036527124 2024-11-19T17:15:31,984 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 018d8d52807e44ffa56725a24796c8b0, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732036527124 2024-11-19T17:15:31,984 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 8df7d19114944f05aa0d06f07aec1f8e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732036528313 2024-11-19T17:15:31,984 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf185c309ff647c8ae42b83989b3e837, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732036528313 2024-11-19T17:15:31,985 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f588ec74f334bab992909bc34527cea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732036529633 2024-11-19T17:15:31,985 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad1d7d8c3b5147ed9187933ed37aadc1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732036529633 2024-11-19T17:15:31,991 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:31,992 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 086da227605bf568b877f13b452b7f9a#B#compaction#391 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:31,993 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/0beee1c46f4542459c723dae80bc0697 is 50, key is test_row_0/B:col10/1732036530321/Put/seqid=0 2024-11-19T17:15:31,994 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111919eb06f63d6a4d26958aaf75c2da8810_086da227605bf568b877f13b452b7f9a store=[table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:31,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742290_1466 (size=12949) 2024-11-19T17:15:31,996 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111919eb06f63d6a4d26958aaf75c2da8810_086da227605bf568b877f13b452b7f9a, store=[table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:31,997 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111919eb06f63d6a4d26958aaf75c2da8810_086da227605bf568b877f13b452b7f9a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:32,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742291_1467 (size=4469) 2024-11-19T17:15:32,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T17:15:32,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:32,038 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 086da227605bf568b877f13b452b7f9a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T17:15:32,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=A 2024-11-19T17:15:32,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:32,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=B 2024-11-19T17:15:32,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:32,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=C 2024-11-19T17:15:32,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:32,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c877dfa542624fb4829cd41e1111a71c_086da227605bf568b877f13b452b7f9a is 50, key is test_row_0/A:col10/1732036530342/Put/seqid=0 2024-11-19T17:15:32,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742292_1468 (size=12454) 2024-11-19T17:15:32,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T17:15:32,401 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/0beee1c46f4542459c723dae80bc0697 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/0beee1c46f4542459c723dae80bc0697 2024-11-19T17:15:32,402 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 086da227605bf568b877f13b452b7f9a#A#compaction#392 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:32,403 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/9a875ae7c8fc4ee2aaea5ef41dccaefa is 175, key is test_row_0/A:col10/1732036530321/Put/seqid=0 2024-11-19T17:15:32,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742293_1469 (size=31903) 2024-11-19T17:15:32,407 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 086da227605bf568b877f13b452b7f9a/B of 086da227605bf568b877f13b452b7f9a into 0beee1c46f4542459c723dae80bc0697(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:32,407 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:32,407 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., storeName=086da227605bf568b877f13b452b7f9a/B, priority=12, startTime=1732036531982; duration=0sec 2024-11-19T17:15:32,407 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:32,407 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 086da227605bf568b877f13b452b7f9a:B 2024-11-19T17:15:32,407 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:15:32,409 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47072 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:15:32,409 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 086da227605bf568b877f13b452b7f9a/C is initiating minor compaction (all files) 2024-11-19T17:15:32,409 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 086da227605bf568b877f13b452b7f9a/C in TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:32,409 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/75917db73f224eea82256fe9dd21587d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/a329770dcdfe4bd48ae54b94944b9a62, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/9402284eb91d4b4a897c8528b4ff673c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/8111d3a2913e4ec68b8751732135b856] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp, totalSize=46.0 K 2024-11-19T17:15:32,410 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 75917db73f224eea82256fe9dd21587d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732036526978 2024-11-19T17:15:32,410 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting a329770dcdfe4bd48ae54b94944b9a62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732036527124 2024-11-19T17:15:32,411 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 9402284eb91d4b4a897c8528b4ff673c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732036528313 2024-11-19T17:15:32,413 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 8111d3a2913e4ec68b8751732135b856, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732036529633 2024-11-19T17:15:32,413 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/9a875ae7c8fc4ee2aaea5ef41dccaefa as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/9a875ae7c8fc4ee2aaea5ef41dccaefa 2024-11-19T17:15:32,419 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 086da227605bf568b877f13b452b7f9a/A of 086da227605bf568b877f13b452b7f9a into 9a875ae7c8fc4ee2aaea5ef41dccaefa(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:32,419 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:32,419 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., storeName=086da227605bf568b877f13b452b7f9a/A, priority=12, startTime=1732036531982; duration=0sec 2024-11-19T17:15:32,419 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:32,419 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 086da227605bf568b877f13b452b7f9a:A 2024-11-19T17:15:32,425 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 086da227605bf568b877f13b452b7f9a#C#compaction#394 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:32,426 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/6f5bd7aa18f04a96a3d2179144e79062 is 50, key is test_row_0/C:col10/1732036530321/Put/seqid=0 2024-11-19T17:15:32,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742294_1470 (size=12949) 2024-11-19T17:15:32,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:32,454 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c877dfa542624fb4829cd41e1111a71c_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c877dfa542624fb4829cd41e1111a71c_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:32,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/bbe3b1f1ac40419b8f115db673c99efe, store: [table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:32,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/bbe3b1f1ac40419b8f115db673c99efe is 175, key is test_row_0/A:col10/1732036530342/Put/seqid=0 2024-11-19T17:15:32,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742295_1471 (size=31255) 2024-11-19T17:15:32,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:32,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:32,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732036532579, max timestamp=1732036532614Num uniques 859; 2024-11-19T17:15:32,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T17:15:32,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732036532579, max timestamp=1732036532614Num uniques 859; 2024-11-19T17:15:32,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732036532579, max timestamp=1732036532614Num uniques 859; 2024-11-19T17:15:32,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T17:15:32,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T17:15:32,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:32,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036592650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:32,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:32,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:32,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:32,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036592652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:32,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:32,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:32,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:32,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036592653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:32,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:32,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:32,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:32,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:32,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036592654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:32,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:32,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:32,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:32,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:32,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036592664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:32,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:32,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:32,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:32,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:32,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036592774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:32,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:32,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036592774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:32,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:32,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:32,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:32,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:32,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:32,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:32,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:32,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036592774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:32,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:32,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:32,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:32,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:32,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036592774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:32,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:32,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:32,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:32,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:32,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036592783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:32,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:32,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:32,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:32,834 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/6f5bd7aa18f04a96a3d2179144e79062 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/6f5bd7aa18f04a96a3d2179144e79062 2024-11-19T17:15:32,838 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 086da227605bf568b877f13b452b7f9a/C of 086da227605bf568b877f13b452b7f9a into 6f5bd7aa18f04a96a3d2179144e79062(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:32,838 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:32,838 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., storeName=086da227605bf568b877f13b452b7f9a/C, priority=12, startTime=1732036531982; duration=0sec 2024-11-19T17:15:32,838 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:32,838 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 086da227605bf568b877f13b452b7f9a:C 2024-11-19T17:15:32,860 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/bbe3b1f1ac40419b8f115db673c99efe 2024-11-19T17:15:32,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/c69eb028f1b148878e487edfd6239ac4 is 50, key is test_row_0/B:col10/1732036530342/Put/seqid=0 2024-11-19T17:15:32,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742296_1472 (size=12301) 2024-11-19T17:15:32,880 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/c69eb028f1b148878e487edfd6239ac4 2024-11-19T17:15:32,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/88c7de26ce264e7183be604e9a7dae30 is 50, key is test_row_0/C:col10/1732036530342/Put/seqid=0 2024-11-19T17:15:32,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742297_1473 (size=12301) 2024-11-19T17:15:32,894 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/88c7de26ce264e7183be604e9a7dae30 2024-11-19T17:15:32,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/bbe3b1f1ac40419b8f115db673c99efe as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/bbe3b1f1ac40419b8f115db673c99efe 2024-11-19T17:15:32,901 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/bbe3b1f1ac40419b8f115db673c99efe, entries=150, sequenceid=323, filesize=30.5 K 2024-11-19T17:15:32,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/c69eb028f1b148878e487edfd6239ac4 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/c69eb028f1b148878e487edfd6239ac4 2024-11-19T17:15:32,906 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/c69eb028f1b148878e487edfd6239ac4, entries=150, sequenceid=323, filesize=12.0 K 2024-11-19T17:15:32,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/88c7de26ce264e7183be604e9a7dae30 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/88c7de26ce264e7183be604e9a7dae30 2024-11-19T17:15:32,910 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/88c7de26ce264e7183be604e9a7dae30, entries=150, sequenceid=323, filesize=12.0 K 2024-11-19T17:15:32,911 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 086da227605bf568b877f13b452b7f9a in 873ms, sequenceid=323, compaction requested=false 2024-11-19T17:15:32,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:32,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:32,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-19T17:15:32,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-19T17:15:32,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-19T17:15:32,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9470 sec 2024-11-19T17:15:32,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.9510 sec 2024-11-19T17:15:33,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:33,001 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 086da227605bf568b877f13b452b7f9a 3/3 column families, dataSize=147.60 KB heapSize=406.08 KB 2024-11-19T17:15:33,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=A 2024-11-19T17:15:33,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:33,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=B 2024-11-19T17:15:33,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:33,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=C 2024-11-19T17:15:33,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:33,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119193f253d6da54b03af985fd91f72c13d_086da227605bf568b877f13b452b7f9a is 50, key is test_row_0/A:col10/1732036532663/Put/seqid=0 2024-11-19T17:15:33,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742298_1474 (size=14994) 2024-11-19T17:15:33,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:33,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036593006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036593004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:33,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036593020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:33,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:33,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:33,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036593021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:33,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:33,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:33,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036593021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:33,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:33,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T17:15:33,114 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:15:33,114 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:15:33,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-19T17:15:33,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T17:15:33,116 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:15:33,117 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:15:33,117 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:15:33,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:33,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036593124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:33,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:33,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036593124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:33,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036593124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:33,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:33,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036593143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:33,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:33,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:33,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036593144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:33,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:33,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T17:15:33,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T17:15:33,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:33,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:33,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:33,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:33,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:33,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:33,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:33,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036593343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:33,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:33,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036593343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:33,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:33,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036593343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:33,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:33,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036593354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:33,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:33,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:33,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036593354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:33,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:33,414 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:33,417 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119193f253d6da54b03af985fd91f72c13d_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119193f253d6da54b03af985fd91f72c13d_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:33,418 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/059d81905cb74cd4b674f7b9a1acefb9, store: [table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:33,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/059d81905cb74cd4b674f7b9a1acefb9 is 175, key is test_row_0/A:col10/1732036532663/Put/seqid=0 2024-11-19T17:15:33,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742299_1475 (size=39949) 2024-11-19T17:15:33,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T17:15:33,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:33,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:33,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:33,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:33,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:33,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:33,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T17:15:33,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T17:15:33,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:33,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:33,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:33,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:33,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:33,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:33,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:33,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036593664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:33,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:33,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036593664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:33,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:33,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036593664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:33,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:33,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036593674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:33,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:33,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:33,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036593674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:33,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:33,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:33,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T17:15:33,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:33,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:33,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:33,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:33,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:33,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:33,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T17:15:33,822 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=361, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/059d81905cb74cd4b674f7b9a1acefb9 2024-11-19T17:15:33,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/52a727de2d11429e93097bdf5a130fc5 is 50, key is test_row_0/B:col10/1732036532663/Put/seqid=0 2024-11-19T17:15:33,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742300_1476 (size=12301) 2024-11-19T17:15:33,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T17:15:33,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:33,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:33,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:33,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:33,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:33,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:34,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T17:15:34,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:34,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:34,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:34,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:34,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:34,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:34,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T17:15:34,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:34,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:34,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:34,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:34,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:34,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:34,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:34,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036594184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:34,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:34,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:34,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036594184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:34,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:34,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:34,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:34,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:34,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036594184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:34,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:34,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:34,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:34,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:34,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036594194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:34,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:34,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:34,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:34,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036594194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:34,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:34,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:34,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:34,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:34,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/52a727de2d11429e93097bdf5a130fc5 2024-11-19T17:15:34,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/e83e5ab7f2394f848b3dd9c732c3b757 is 50, key is test_row_0/C:col10/1732036532663/Put/seqid=0 2024-11-19T17:15:34,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742301_1477 (size=12301) 2024-11-19T17:15:34,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T17:15:34,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T17:15:34,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:34,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:34,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:34,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:34,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:34,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:34,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T17:15:34,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:34,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:34,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:34,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:34,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:34,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:34,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/e83e5ab7f2394f848b3dd9c732c3b757 2024-11-19T17:15:34,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T17:15:34,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:34,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:34,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:34,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:34,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:34,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:34,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/059d81905cb74cd4b674f7b9a1acefb9 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/059d81905cb74cd4b674f7b9a1acefb9 2024-11-19T17:15:34,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/059d81905cb74cd4b674f7b9a1acefb9, entries=200, sequenceid=361, filesize=39.0 K 2024-11-19T17:15:34,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/52a727de2d11429e93097bdf5a130fc5 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/52a727de2d11429e93097bdf5a130fc5 2024-11-19T17:15:34,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/52a727de2d11429e93097bdf5a130fc5, entries=150, sequenceid=361, filesize=12.0 K 2024-11-19T17:15:34,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/e83e5ab7f2394f848b3dd9c732c3b757 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/e83e5ab7f2394f848b3dd9c732c3b757 2024-11-19T17:15:34,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/e83e5ab7f2394f848b3dd9c732c3b757, entries=150, sequenceid=361, filesize=12.0 K 2024-11-19T17:15:34,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~406.03 KB/415776, currentSize=40.25 KB/41220 for 086da227605bf568b877f13b452b7f9a in 1662ms, sequenceid=361, compaction requested=true 2024-11-19T17:15:34,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:34,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 086da227605bf568b877f13b452b7f9a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:15:34,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:34,663 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:34,663 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:34,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 086da227605bf568b877f13b452b7f9a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:15:34,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:34,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 086da227605bf568b877f13b452b7f9a:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:15:34,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:34,664 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:34,664 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:34,664 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 086da227605bf568b877f13b452b7f9a/B is initiating minor compaction (all files) 2024-11-19T17:15:34,664 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 086da227605bf568b877f13b452b7f9a/A is initiating minor compaction (all files) 2024-11-19T17:15:34,664 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 086da227605bf568b877f13b452b7f9a/B in TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:34,664 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 086da227605bf568b877f13b452b7f9a/A in TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:34,665 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/0beee1c46f4542459c723dae80bc0697, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/c69eb028f1b148878e487edfd6239ac4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/52a727de2d11429e93097bdf5a130fc5] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp, totalSize=36.7 K 2024-11-19T17:15:34,665 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/9a875ae7c8fc4ee2aaea5ef41dccaefa, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/bbe3b1f1ac40419b8f115db673c99efe, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/059d81905cb74cd4b674f7b9a1acefb9] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp, totalSize=100.7 K 2024-11-19T17:15:34,665 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:34,665 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. files: [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/9a875ae7c8fc4ee2aaea5ef41dccaefa, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/bbe3b1f1ac40419b8f115db673c99efe, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/059d81905cb74cd4b674f7b9a1acefb9] 2024-11-19T17:15:34,665 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 0beee1c46f4542459c723dae80bc0697, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732036529633 2024-11-19T17:15:34,665 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a875ae7c8fc4ee2aaea5ef41dccaefa, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732036529633 2024-11-19T17:15:34,666 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting c69eb028f1b148878e487edfd6239ac4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732036530326 2024-11-19T17:15:34,666 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbe3b1f1ac40419b8f115db673c99efe, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732036530326 2024-11-19T17:15:34,666 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 52a727de2d11429e93097bdf5a130fc5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732036532650 2024-11-19T17:15:34,667 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 059d81905cb74cd4b674f7b9a1acefb9, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732036532650 2024-11-19T17:15:34,674 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 086da227605bf568b877f13b452b7f9a#B#compaction#400 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:34,674 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:34,674 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/9846884075324235a7638337bfad8ad3 is 50, key is test_row_0/B:col10/1732036532663/Put/seqid=0 2024-11-19T17:15:34,680 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411195ccebda7c078454390d7c31811d64001_086da227605bf568b877f13b452b7f9a store=[table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:34,682 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411195ccebda7c078454390d7c31811d64001_086da227605bf568b877f13b452b7f9a, store=[table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:34,682 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195ccebda7c078454390d7c31811d64001_086da227605bf568b877f13b452b7f9a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:34,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742302_1478 (size=13051) 2024-11-19T17:15:34,692 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/9846884075324235a7638337bfad8ad3 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/9846884075324235a7638337bfad8ad3 2024-11-19T17:15:34,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742303_1479 (size=4469) 2024-11-19T17:15:34,701 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 086da227605bf568b877f13b452b7f9a#A#compaction#401 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:34,701 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/22eb00ae8bd6420fa01834b29c3b71f8 is 175, key is test_row_0/A:col10/1732036532663/Put/seqid=0 2024-11-19T17:15:34,702 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 086da227605bf568b877f13b452b7f9a/B of 086da227605bf568b877f13b452b7f9a into 9846884075324235a7638337bfad8ad3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:34,702 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:34,702 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., storeName=086da227605bf568b877f13b452b7f9a/B, priority=13, startTime=1732036534663; duration=0sec 2024-11-19T17:15:34,703 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:34,703 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 086da227605bf568b877f13b452b7f9a:B 2024-11-19T17:15:34,703 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:34,706 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:34,706 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 086da227605bf568b877f13b452b7f9a/C is initiating minor compaction (all files) 2024-11-19T17:15:34,706 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 086da227605bf568b877f13b452b7f9a/C in TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:34,706 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/6f5bd7aa18f04a96a3d2179144e79062, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/88c7de26ce264e7183be604e9a7dae30, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/e83e5ab7f2394f848b3dd9c732c3b757] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp, totalSize=36.7 K 2024-11-19T17:15:34,706 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f5bd7aa18f04a96a3d2179144e79062, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732036529633 2024-11-19T17:15:34,707 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 88c7de26ce264e7183be604e9a7dae30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732036530326 2024-11-19T17:15:34,707 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting e83e5ab7f2394f848b3dd9c732c3b757, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732036532650 2024-11-19T17:15:34,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742304_1480 (size=32005) 2024-11-19T17:15:34,716 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/22eb00ae8bd6420fa01834b29c3b71f8 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/22eb00ae8bd6420fa01834b29c3b71f8 2024-11-19T17:15:34,720 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 086da227605bf568b877f13b452b7f9a#C#compaction#402 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:34,721 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/c5988174daf64eeca2d3c93e1d62796b is 50, key is test_row_0/C:col10/1732036532663/Put/seqid=0 2024-11-19T17:15:34,722 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 086da227605bf568b877f13b452b7f9a/A of 086da227605bf568b877f13b452b7f9a into 22eb00ae8bd6420fa01834b29c3b71f8(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:34,722 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:34,722 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., storeName=086da227605bf568b877f13b452b7f9a/A, priority=13, startTime=1732036534663; duration=0sec 2024-11-19T17:15:34,722 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:34,722 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 086da227605bf568b877f13b452b7f9a:A 2024-11-19T17:15:34,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742305_1481 (size=13051) 2024-11-19T17:15:34,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T17:15:34,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:34,800 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 086da227605bf568b877f13b452b7f9a 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-19T17:15:34,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=A 2024-11-19T17:15:34,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:34,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=B 2024-11-19T17:15:34,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:34,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=C 2024-11-19T17:15:34,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:34,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111985b36af1c0684c43b6c75beb4323ba69_086da227605bf568b877f13b452b7f9a is 50, key is test_row_0/A:col10/1732036533018/Put/seqid=0 2024-11-19T17:15:34,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742306_1482 (size=12454) 2024-11-19T17:15:35,129 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/c5988174daf64eeca2d3c93e1d62796b as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/c5988174daf64eeca2d3c93e1d62796b 2024-11-19T17:15:35,133 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 086da227605bf568b877f13b452b7f9a/C of 086da227605bf568b877f13b452b7f9a into c5988174daf64eeca2d3c93e1d62796b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:35,133 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:35,133 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., storeName=086da227605bf568b877f13b452b7f9a/C, priority=13, startTime=1732036534664; duration=0sec 2024-11-19T17:15:35,133 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:35,133 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 086da227605bf568b877f13b452b7f9a:C 2024-11-19T17:15:35,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:35,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. as already flushing 2024-11-19T17:15:35,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:35,214 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111985b36af1c0684c43b6c75beb4323ba69_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111985b36af1c0684c43b6c75beb4323ba69_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:35,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/9a1aeee6c4ce458a893e9e5379afa444, store: [table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:35,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/9a1aeee6c4ce458a893e9e5379afa444 is 175, key is test_row_0/A:col10/1732036533018/Put/seqid=0 2024-11-19T17:15:35,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742307_1483 (size=31255) 2024-11-19T17:15:35,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036535237, max timestamp=1732036535249Num uniques 859; 2024-11-19T17:15:35,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036535237, max timestamp=1732036535249Num uniques 859; 2024-11-19T17:15:35,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:15:35,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:15:35,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036535237, max timestamp=1732036535249Num uniques 859; 2024-11-19T17:15:35,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:15:35,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T17:15:35,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:35,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036595287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:35,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:35,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036595288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:35,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:35,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:35,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732036535282, max timestamp=1732036535299Num uniques 1018; 2024-11-19T17:15:35,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T17:15:35,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732036535282, max timestamp=1732036535299Num uniques 1018; 2024-11-19T17:15:35,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T17:15:35,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732036535282, max timestamp=1732036535299Num uniques 1018; 2024-11-19T17:15:35,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T17:15:35,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732036535360, max timestamp=1732036535385Num uniques 1077; 2024-11-19T17:15:35,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-19T17:15:35,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732036535360, max timestamp=1732036535385Num uniques 1077; 2024-11-19T17:15:35,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-19T17:15:35,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732036535360, max timestamp=1732036535385Num uniques 1077; 2024-11-19T17:15:35,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-19T17:15:35,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732036535423, max timestamp=1732036535445Num uniques 1086; 2024-11-19T17:15:35,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732036535423, max timestamp=1732036535445Num uniques 1086; 2024-11-19T17:15:35,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-19T17:15:35,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-19T17:15:35,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732036535423, max timestamp=1732036535445Num uniques 1086; 2024-11-19T17:15:35,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-19T17:15:35,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:35,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:35,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036595479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:35,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036595483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:35,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:35,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036595511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:35,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:35,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036595511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:35,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:35,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:35,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:35,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036595511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:35,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:35,619 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=372, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/9a1aeee6c4ce458a893e9e5379afa444 2024-11-19T17:15:35,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:35,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/5d54f0f7770c482ab450f9e67bd440c0 is 50, key is test_row_0/B:col10/1732036533018/Put/seqid=0 2024-11-19T17:15:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036595614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:35,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:35,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036595614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:35,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:35,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:35,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742308_1484 (size=12301) 2024-11-19T17:15:35,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:35,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036595633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:35,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:35,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036595633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:35,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:35,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:35,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036595634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:35,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:35,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:35,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036595833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:35,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036595833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:35,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036595853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:35,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:35,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036595854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:35,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036595854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:36,022 INFO [Thread-1756 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1756. 2024-11-19T17:15:36,022 DEBUG [Thread-1756 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:15:36,022 DEBUG [Thread-1756 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:36,022 DEBUG [Thread-1756 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:36,022 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:15:36,025 INFO [Thread-1760 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1760. 2024-11-19T17:15:36,026 DEBUG [Thread-1760 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:15:36,026 DEBUG [Thread-1760 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:36,026 DEBUG [Thread-1760 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:36,026 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:15:36,028 INFO [Thread-1762 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1762. 2024-11-19T17:15:36,028 DEBUG [Thread-1762 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:15:36,028 DEBUG [Thread-1762 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:36,028 DEBUG [Thread-1762 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:36,028 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:15:36,033 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/5d54f0f7770c482ab450f9e67bd440c0 2024-11-19T17:15:36,034 INFO [Thread-1758 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1758. 2024-11-19T17:15:36,034 DEBUG [Thread-1758 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:15:36,034 DEBUG [Thread-1758 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:36,034 DEBUG [Thread-1758 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:36,034 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:15:36,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/372b3dc7b90f402fbf4a65e83355c97e is 50, key is test_row_0/C:col10/1732036533018/Put/seqid=0 2024-11-19T17:15:36,039 INFO [Thread-1754 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1754. 2024-11-19T17:15:36,039 DEBUG [Thread-1754 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:15:36,039 DEBUG [Thread-1754 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:36,040 DEBUG [Thread-1754 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:36,040 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:15:36,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742309_1485 (size=12301) 2024-11-19T17:15:36,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:36,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:36,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46906 deadline: 1732036596154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:36,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46952 deadline: 1732036596154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:36,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:36,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:36,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:36,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:36,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:36,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:36,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:36,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732036596173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:36,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:36,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:36,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46984 deadline: 1732036596173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:36,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46918 deadline: 1732036596174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:36,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:36,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:36,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:36,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:36,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:36,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:36,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:36,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:36,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:15:36,442 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/372b3dc7b90f402fbf4a65e83355c97e 2024-11-19T17:15:36,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/9a1aeee6c4ce458a893e9e5379afa444 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/9a1aeee6c4ce458a893e9e5379afa444 2024-11-19T17:15:36,449 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/9a1aeee6c4ce458a893e9e5379afa444, entries=150, sequenceid=372, filesize=30.5 K 2024-11-19T17:15:36,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/5d54f0f7770c482ab450f9e67bd440c0 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/5d54f0f7770c482ab450f9e67bd440c0 2024-11-19T17:15:36,452 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/5d54f0f7770c482ab450f9e67bd440c0, entries=150, sequenceid=372, filesize=12.0 K 2024-11-19T17:15:36,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/372b3dc7b90f402fbf4a65e83355c97e as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/372b3dc7b90f402fbf4a65e83355c97e 2024-11-19T17:15:36,455 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/372b3dc7b90f402fbf4a65e83355c97e, entries=150, sequenceid=372, filesize=12.0 K 2024-11-19T17:15:36,456 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 086da227605bf568b877f13b452b7f9a in 1657ms, sequenceid=372, compaction requested=false 2024-11-19T17:15:36,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:36,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:36,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-19T17:15:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-19T17:15:36,459 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-19T17:15:36,459 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3410 sec 2024-11-19T17:15:36,461 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 3.3450 sec 2024-11-19T17:15:36,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:36,666 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 086da227605bf568b877f13b452b7f9a 3/3 column families, dataSize=147.60 KB heapSize=413.16 KB 2024-11-19T17:15:36,666 INFO [Thread-1751 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1751. 2024-11-19T17:15:36,666 DEBUG [Thread-1751 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:15:36,666 DEBUG [Thread-1751 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:36,666 DEBUG [Thread-1751 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:36,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=A 2024-11-19T17:15:36,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T17:15:36,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=B 2024-11-19T17:15:36,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T17:15:36,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=C 2024-11-19T17:15:36,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T17:15:36,666 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:15:36,667 INFO [Thread-1743 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1743. 2024-11-19T17:15:36,667 DEBUG [Thread-1743 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:15:36,667 DEBUG [Thread-1743 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:36,667 DEBUG [Thread-1743 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:36,667 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:15:36,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d683399c870140ccb198bb224ad234f5_086da227605bf568b877f13b452b7f9a is 50, key is test_row_0/A:col10/1732036536664/Put/seqid=0 2024-11-19T17:15:36,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742310_1486 (size=12454) 2024-11-19T17:15:36,685 INFO [Thread-1749 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1749. 2024-11-19T17:15:36,685 INFO [Thread-1747 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1747. 2024-11-19T17:15:36,685 DEBUG [Thread-1749 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:15:36,685 DEBUG [Thread-1747 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:15:36,685 DEBUG [Thread-1749 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:36,685 DEBUG [Thread-1747 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:36,685 INFO [Thread-1745 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1745. 2024-11-19T17:15:36,685 DEBUG [Thread-1749 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:36,685 DEBUG [Thread-1747 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:36,685 DEBUG [Thread-1745 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:15:36,685 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:15:36,685 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:15:36,685 DEBUG [Thread-1745 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:36,685 DEBUG [Thread-1745 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:36,685 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:15:37,076 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:37,079 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d683399c870140ccb198bb224ad234f5_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d683399c870140ccb198bb224ad234f5_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:37,079 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/a121dd95771c43a4a7cd8d7bf2ecd6a8, store: [table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:37,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/a121dd95771c43a4a7cd8d7bf2ecd6a8 is 175, key is test_row_0/A:col10/1732036536664/Put/seqid=0 2024-11-19T17:15:37,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742311_1487 (size=31255) 2024-11-19T17:15:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T17:15:37,284 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:15:37,284 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T17:15:37,284 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-19T17:15:37,284 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-19T17:15:37,284 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-19T17:15:37,284 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-19T17:15:37,284 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-19T17:15:37,284 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T17:15:37,284 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T17:15:37,284 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 946 2024-11-19T17:15:37,284 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2836 rows 2024-11-19T17:15:37,284 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 954 2024-11-19T17:15:37,284 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2859 rows 2024-11-19T17:15:37,284 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 950 2024-11-19T17:15:37,284 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2847 rows 2024-11-19T17:15:37,284 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 951 2024-11-19T17:15:37,284 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2853 rows 2024-11-19T17:15:37,284 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 949 2024-11-19T17:15:37,284 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2846 rows 2024-11-19T17:15:37,284 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T17:15:37,284 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T17:15:37,285 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:37,285 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:37,285 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:15:37,285 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T17:15:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T17:15:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T17:15:37,288 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732036537288"}]},"ts":"1732036537288"} 2024-11-19T17:15:37,289 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T17:15:37,289 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T17:15:37,290 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T17:15:37,291 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=086da227605bf568b877f13b452b7f9a, UNASSIGN}] 2024-11-19T17:15:37,292 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=086da227605bf568b877f13b452b7f9a, UNASSIGN 2024-11-19T17:15:37,292 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=086da227605bf568b877f13b452b7f9a, regionState=CLOSING, regionLocation=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:37,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=086da227605bf568b877f13b452b7f9a, UNASSIGN because future has completed 2024-11-19T17:15:37,293 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T17:15:37,294 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; CloseRegionProcedure 086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473}] 2024-11-19T17:15:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T17:15:37,445 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] handler.UnassignRegionHandler(122): Close 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:37,445 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T17:15:37,445 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1722): Closing 086da227605bf568b877f13b452b7f9a, disabling compactions & flushes 2024-11-19T17:15:37,446 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:37,483 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=462, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/a121dd95771c43a4a7cd8d7bf2ecd6a8 2024-11-19T17:15:37,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/5b9c24cdfa844dcca73cf9a456840f8a is 50, key is test_row_0/B:col10/1732036536664/Put/seqid=0 2024-11-19T17:15:37,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742312_1488 (size=12301) 2024-11-19T17:15:37,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T17:15:37,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/5b9c24cdfa844dcca73cf9a456840f8a 2024-11-19T17:15:37,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/328ef6571ec8472e8c60423f148d8521 is 50, key is test_row_0/C:col10/1732036536664/Put/seqid=0 2024-11-19T17:15:37,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742313_1489 (size=12301) 2024-11-19T17:15:37,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T17:15:38,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/328ef6571ec8472e8c60423f148d8521 2024-11-19T17:15:38,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/a121dd95771c43a4a7cd8d7bf2ecd6a8 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/a121dd95771c43a4a7cd8d7bf2ecd6a8 2024-11-19T17:15:38,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/a121dd95771c43a4a7cd8d7bf2ecd6a8, entries=150, sequenceid=462, filesize=30.5 K 2024-11-19T17:15:38,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/5b9c24cdfa844dcca73cf9a456840f8a as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/5b9c24cdfa844dcca73cf9a456840f8a 2024-11-19T17:15:38,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/5b9c24cdfa844dcca73cf9a456840f8a, entries=150, sequenceid=462, filesize=12.0 K 2024-11-19T17:15:38,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/328ef6571ec8472e8c60423f148d8521 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/328ef6571ec8472e8c60423f148d8521 2024-11-19T17:15:38,314 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/328ef6571ec8472e8c60423f148d8521, entries=150, sequenceid=462, filesize=12.0 K 2024-11-19T17:15:38,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~433.50 KB/443904, currentSize=20.13 KB/20610 for 086da227605bf568b877f13b452b7f9a in 1649ms, sequenceid=462, compaction requested=true 2024-11-19T17:15:38,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 086da227605bf568b877f13b452b7f9a: 2024-11-19T17:15:38,315 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:38,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 086da227605bf568b877f13b452b7f9a:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:15:38,315 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:38,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:38,315 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. because compaction request was cancelled 2024-11-19T17:15:38,315 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. after waiting 0 ms 2024-11-19T17:15:38,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 086da227605bf568b877f13b452b7f9a:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:15:38,315 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:38,315 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 086da227605bf568b877f13b452b7f9a:A 2024-11-19T17:15:38,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:38,315 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. because compaction request was cancelled 2024-11-19T17:15:38,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 086da227605bf568b877f13b452b7f9a:C, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:15:38,315 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. because compaction request was cancelled 2024-11-19T17:15:38,315 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 086da227605bf568b877f13b452b7f9a:B 2024-11-19T17:15:38,315 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 086da227605bf568b877f13b452b7f9a:C 2024-11-19T17:15:38,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:38,315 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(2902): Flushing 086da227605bf568b877f13b452b7f9a 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-19T17:15:38,315 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=A 2024-11-19T17:15:38,315 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:38,315 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=B 2024-11-19T17:15:38,315 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:38,315 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 086da227605bf568b877f13b452b7f9a, store=C 2024-11-19T17:15:38,315 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:38,319 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e7be1e51452b4004ad19146e9b4a5cfd_086da227605bf568b877f13b452b7f9a is 50, key is test_row_0/A:col10/1732036536684/Put/seqid=0 2024-11-19T17:15:38,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742314_1490 (size=12454) 2024-11-19T17:15:38,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T17:15:38,723 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:38,726 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e7be1e51452b4004ad19146e9b4a5cfd_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e7be1e51452b4004ad19146e9b4a5cfd_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:38,727 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/dc4b998f5a8e472e9d0a85b5a50d48f7, store: [table=TestAcidGuarantees family=A region=086da227605bf568b877f13b452b7f9a] 2024-11-19T17:15:38,727 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/dc4b998f5a8e472e9d0a85b5a50d48f7 is 175, key is test_row_0/A:col10/1732036536684/Put/seqid=0 2024-11-19T17:15:38,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742315_1491 (size=31255) 2024-11-19T17:15:39,131 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=468, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/dc4b998f5a8e472e9d0a85b5a50d48f7 2024-11-19T17:15:39,136 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/653118ddab4848e1b1156122fe803d5c is 50, key is test_row_0/B:col10/1732036536684/Put/seqid=0 2024-11-19T17:15:39,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742316_1492 (size=12301) 2024-11-19T17:15:39,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T17:15:39,540 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/653118ddab4848e1b1156122fe803d5c 2024-11-19T17:15:39,545 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/6099ec04e6b64b529b251b8fe6a7158f is 50, key is test_row_0/C:col10/1732036536684/Put/seqid=0 2024-11-19T17:15:39,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742317_1493 (size=12301) 2024-11-19T17:15:39,949 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/6099ec04e6b64b529b251b8fe6a7158f 2024-11-19T17:15:39,952 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/A/dc4b998f5a8e472e9d0a85b5a50d48f7 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/dc4b998f5a8e472e9d0a85b5a50d48f7 2024-11-19T17:15:39,954 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/dc4b998f5a8e472e9d0a85b5a50d48f7, entries=150, sequenceid=468, filesize=30.5 K 2024-11-19T17:15:39,955 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/B/653118ddab4848e1b1156122fe803d5c as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/653118ddab4848e1b1156122fe803d5c 2024-11-19T17:15:39,957 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/653118ddab4848e1b1156122fe803d5c, entries=150, sequenceid=468, filesize=12.0 K 2024-11-19T17:15:39,958 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/.tmp/C/6099ec04e6b64b529b251b8fe6a7158f as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/6099ec04e6b64b529b251b8fe6a7158f 2024-11-19T17:15:39,960 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/6099ec04e6b64b529b251b8fe6a7158f, entries=150, sequenceid=468, filesize=12.0 K 2024-11-19T17:15:39,961 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 086da227605bf568b877f13b452b7f9a in 1646ms, sequenceid=468, compaction requested=true 2024-11-19T17:15:39,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/845c360d9cee4d4985c37191abfde3e0, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/b3bc6c4525e44c559357f5e1126dae4a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/65b085eb43484f509735372e871e58ec, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/1d4d80b2b4904d6498731ddb87e6a342, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/e638c055f90142b497eb27b142bd3b94, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/0537babac0f84579a70afac6728889b7, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/ca8cb6de3bd24fdca019262443c5774d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/1751ef27232f40b99991bc8ba92555ae, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/63f4bee02aaa4a8688d96b8dd955c95b, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/9b6d806afe3d4195bd437b6556d1c97a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/b78bad7f0a3b42b9b909d0d2e78b4f14, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/7f7accba6d714a648f5e5fb2dbfa8dda, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/44212c4215bc478095d0d3da90270482, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/69d92541c11d432f897ad4f9cdc28af9, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/7e1a7b8eb331452baf7b606829f5c886, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/8da0e1421e574f9caed7b49433baa2fe, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/018d8d52807e44ffa56725a24796c8b0, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/bf185c309ff647c8ae42b83989b3e837, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/ad1d7d8c3b5147ed9187933ed37aadc1, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/9a875ae7c8fc4ee2aaea5ef41dccaefa, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/bbe3b1f1ac40419b8f115db673c99efe, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/059d81905cb74cd4b674f7b9a1acefb9] to archive 2024-11-19T17:15:39,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:15:39,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/845c360d9cee4d4985c37191abfde3e0 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/845c360d9cee4d4985c37191abfde3e0 2024-11-19T17:15:39,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/b3bc6c4525e44c559357f5e1126dae4a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/b3bc6c4525e44c559357f5e1126dae4a 2024-11-19T17:15:39,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/65b085eb43484f509735372e871e58ec to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/65b085eb43484f509735372e871e58ec 2024-11-19T17:15:39,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/1d4d80b2b4904d6498731ddb87e6a342 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/1d4d80b2b4904d6498731ddb87e6a342 2024-11-19T17:15:39,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/e638c055f90142b497eb27b142bd3b94 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/e638c055f90142b497eb27b142bd3b94 2024-11-19T17:15:39,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/0537babac0f84579a70afac6728889b7 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/0537babac0f84579a70afac6728889b7 2024-11-19T17:15:39,968 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/ca8cb6de3bd24fdca019262443c5774d to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/ca8cb6de3bd24fdca019262443c5774d 2024-11-19T17:15:39,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/1751ef27232f40b99991bc8ba92555ae to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/1751ef27232f40b99991bc8ba92555ae 2024-11-19T17:15:39,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/63f4bee02aaa4a8688d96b8dd955c95b to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/63f4bee02aaa4a8688d96b8dd955c95b 2024-11-19T17:15:39,970 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/9b6d806afe3d4195bd437b6556d1c97a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/9b6d806afe3d4195bd437b6556d1c97a 2024-11-19T17:15:39,971 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/b78bad7f0a3b42b9b909d0d2e78b4f14 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/b78bad7f0a3b42b9b909d0d2e78b4f14 2024-11-19T17:15:39,972 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/7f7accba6d714a648f5e5fb2dbfa8dda to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/7f7accba6d714a648f5e5fb2dbfa8dda 2024-11-19T17:15:39,972 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/44212c4215bc478095d0d3da90270482 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/44212c4215bc478095d0d3da90270482 2024-11-19T17:15:39,973 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/69d92541c11d432f897ad4f9cdc28af9 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/69d92541c11d432f897ad4f9cdc28af9 2024-11-19T17:15:39,974 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/7e1a7b8eb331452baf7b606829f5c886 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/7e1a7b8eb331452baf7b606829f5c886 2024-11-19T17:15:39,975 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/8da0e1421e574f9caed7b49433baa2fe to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/8da0e1421e574f9caed7b49433baa2fe 2024-11-19T17:15:39,975 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/018d8d52807e44ffa56725a24796c8b0 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/018d8d52807e44ffa56725a24796c8b0 2024-11-19T17:15:39,976 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/bf185c309ff647c8ae42b83989b3e837 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/bf185c309ff647c8ae42b83989b3e837 2024-11-19T17:15:39,977 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/ad1d7d8c3b5147ed9187933ed37aadc1 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/ad1d7d8c3b5147ed9187933ed37aadc1 2024-11-19T17:15:39,978 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/9a875ae7c8fc4ee2aaea5ef41dccaefa to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/9a875ae7c8fc4ee2aaea5ef41dccaefa 2024-11-19T17:15:39,978 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/bbe3b1f1ac40419b8f115db673c99efe to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/bbe3b1f1ac40419b8f115db673c99efe 2024-11-19T17:15:39,979 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/059d81905cb74cd4b674f7b9a1acefb9 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/059d81905cb74cd4b674f7b9a1acefb9 2024-11-19T17:15:39,980 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/fbd23b0c49494ba5b2c1f7f2edef84b4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/1d8d8f2416c2447494f30e214c2c64cb, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/66afa27d971b4a758dfedaf8c69453ec, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/fbe9652e4c40438082d1ff4cf6864369, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/9170c2d60f87451d8f4780b38b419e9d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/ddb76d7d087d4623a91d1757f72d0fff, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/fdc7481dbd014bf6bbe74e814768936f, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/8d1915ab82b245db9bd55d6724bd7b4f, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/ddcccbf0b5da4904985fffab741f0a6d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/1ea03b9eaa7f4eb39c6f99fc12f006fa, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/9a5cf4e74ede420c9b08d3124f92bb2e, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/05a8f002ff644661b11a9d974fbe8039, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/3c6a5ca5b433441181b7aa17be9ebc29, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/c01ecc84631e49a897dca99b174a7a2e, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/976eb33c834149b9a0ab268a415f1cb9, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/7294b2c8bd6b47bdb6067c858c9ce657, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/93129d8219f8478196ced2c66af3a038, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/8df7d19114944f05aa0d06f07aec1f8e, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/0beee1c46f4542459c723dae80bc0697, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/3f588ec74f334bab992909bc34527cea, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/c69eb028f1b148878e487edfd6239ac4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/52a727de2d11429e93097bdf5a130fc5] to archive 2024-11-19T17:15:39,981 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:15:39,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/fbd23b0c49494ba5b2c1f7f2edef84b4 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/fbd23b0c49494ba5b2c1f7f2edef84b4 2024-11-19T17:15:39,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/1d8d8f2416c2447494f30e214c2c64cb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/1d8d8f2416c2447494f30e214c2c64cb 2024-11-19T17:15:39,984 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/66afa27d971b4a758dfedaf8c69453ec to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/66afa27d971b4a758dfedaf8c69453ec 2024-11-19T17:15:39,984 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/fbe9652e4c40438082d1ff4cf6864369 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/fbe9652e4c40438082d1ff4cf6864369 2024-11-19T17:15:39,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/9170c2d60f87451d8f4780b38b419e9d to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/9170c2d60f87451d8f4780b38b419e9d 2024-11-19T17:15:39,986 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/ddb76d7d087d4623a91d1757f72d0fff to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/ddb76d7d087d4623a91d1757f72d0fff 2024-11-19T17:15:39,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/fdc7481dbd014bf6bbe74e814768936f to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/fdc7481dbd014bf6bbe74e814768936f 2024-11-19T17:15:39,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/8d1915ab82b245db9bd55d6724bd7b4f to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/8d1915ab82b245db9bd55d6724bd7b4f 2024-11-19T17:15:39,988 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/ddcccbf0b5da4904985fffab741f0a6d to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/ddcccbf0b5da4904985fffab741f0a6d 2024-11-19T17:15:39,989 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/1ea03b9eaa7f4eb39c6f99fc12f006fa to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/1ea03b9eaa7f4eb39c6f99fc12f006fa 2024-11-19T17:15:39,989 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/9a5cf4e74ede420c9b08d3124f92bb2e to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/9a5cf4e74ede420c9b08d3124f92bb2e 2024-11-19T17:15:39,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/05a8f002ff644661b11a9d974fbe8039 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/05a8f002ff644661b11a9d974fbe8039 2024-11-19T17:15:39,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/3c6a5ca5b433441181b7aa17be9ebc29 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/3c6a5ca5b433441181b7aa17be9ebc29 2024-11-19T17:15:39,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/c01ecc84631e49a897dca99b174a7a2e to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/c01ecc84631e49a897dca99b174a7a2e 2024-11-19T17:15:39,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/976eb33c834149b9a0ab268a415f1cb9 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/976eb33c834149b9a0ab268a415f1cb9 2024-11-19T17:15:39,993 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/7294b2c8bd6b47bdb6067c858c9ce657 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/7294b2c8bd6b47bdb6067c858c9ce657 2024-11-19T17:15:39,994 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/93129d8219f8478196ced2c66af3a038 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/93129d8219f8478196ced2c66af3a038 2024-11-19T17:15:39,995 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/8df7d19114944f05aa0d06f07aec1f8e to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/8df7d19114944f05aa0d06f07aec1f8e 2024-11-19T17:15:39,995 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/0beee1c46f4542459c723dae80bc0697 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/0beee1c46f4542459c723dae80bc0697 2024-11-19T17:15:39,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/3f588ec74f334bab992909bc34527cea to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/3f588ec74f334bab992909bc34527cea 2024-11-19T17:15:39,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/c69eb028f1b148878e487edfd6239ac4 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/c69eb028f1b148878e487edfd6239ac4 2024-11-19T17:15:39,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/52a727de2d11429e93097bdf5a130fc5 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/52a727de2d11429e93097bdf5a130fc5 2024-11-19T17:15:39,999 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/0fc3ac0534584d1fbcbd133ca9730388, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/34f6e1e3741b45eeb7d64631e4565897, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/6f808c493e0e442e8961762552bed051, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/86cb26ae4caf45a1a894f02fd4d9219b, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/c67e6f9ca7344a4eb621bc734e502ed3, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/8443768e83684dffbf9cbb176bc8fc80, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/761c5122ad4b412d82d0efcb133c6903, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/2371f09d2e0348f39464c34bad3f2477, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/22ba8c8e66c849e698b19ba4ff7d83c4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/491f707643184edc85e95a0b43a66a54, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/fa7aa08a67fa41d9a3a37c3625ed71c7, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/ccc6364e42154eb086e4a656de1f8944, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/e3ae8aa4ca12481cae59a074efa1c336, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/40f91a2a896a473597f63bea3134495a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/75917db73f224eea82256fe9dd21587d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/d6ac24ee8e334e7ea0b5d002992fce41, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/a329770dcdfe4bd48ae54b94944b9a62, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/9402284eb91d4b4a897c8528b4ff673c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/6f5bd7aa18f04a96a3d2179144e79062, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/8111d3a2913e4ec68b8751732135b856, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/88c7de26ce264e7183be604e9a7dae30, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/e83e5ab7f2394f848b3dd9c732c3b757] to archive 2024-11-19T17:15:40,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:15:40,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/0fc3ac0534584d1fbcbd133ca9730388 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/0fc3ac0534584d1fbcbd133ca9730388 2024-11-19T17:15:40,002 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/34f6e1e3741b45eeb7d64631e4565897 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/34f6e1e3741b45eeb7d64631e4565897 2024-11-19T17:15:40,002 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/6f808c493e0e442e8961762552bed051 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/6f808c493e0e442e8961762552bed051 2024-11-19T17:15:40,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/86cb26ae4caf45a1a894f02fd4d9219b to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/86cb26ae4caf45a1a894f02fd4d9219b 2024-11-19T17:15:40,004 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/c67e6f9ca7344a4eb621bc734e502ed3 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/c67e6f9ca7344a4eb621bc734e502ed3 2024-11-19T17:15:40,005 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/8443768e83684dffbf9cbb176bc8fc80 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/8443768e83684dffbf9cbb176bc8fc80 2024-11-19T17:15:40,005 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/761c5122ad4b412d82d0efcb133c6903 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/761c5122ad4b412d82d0efcb133c6903 2024-11-19T17:15:40,006 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/2371f09d2e0348f39464c34bad3f2477 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/2371f09d2e0348f39464c34bad3f2477 2024-11-19T17:15:40,007 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/22ba8c8e66c849e698b19ba4ff7d83c4 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/22ba8c8e66c849e698b19ba4ff7d83c4 2024-11-19T17:15:40,007 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/491f707643184edc85e95a0b43a66a54 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/491f707643184edc85e95a0b43a66a54 2024-11-19T17:15:40,008 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/fa7aa08a67fa41d9a3a37c3625ed71c7 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/fa7aa08a67fa41d9a3a37c3625ed71c7 2024-11-19T17:15:40,009 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/ccc6364e42154eb086e4a656de1f8944 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/ccc6364e42154eb086e4a656de1f8944 2024-11-19T17:15:40,009 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/e3ae8aa4ca12481cae59a074efa1c336 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/e3ae8aa4ca12481cae59a074efa1c336 2024-11-19T17:15:40,010 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/40f91a2a896a473597f63bea3134495a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/40f91a2a896a473597f63bea3134495a 2024-11-19T17:15:40,011 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/75917db73f224eea82256fe9dd21587d to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/75917db73f224eea82256fe9dd21587d 2024-11-19T17:15:40,011 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/d6ac24ee8e334e7ea0b5d002992fce41 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/d6ac24ee8e334e7ea0b5d002992fce41 2024-11-19T17:15:40,012 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/a329770dcdfe4bd48ae54b94944b9a62 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/a329770dcdfe4bd48ae54b94944b9a62 2024-11-19T17:15:40,013 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/9402284eb91d4b4a897c8528b4ff673c to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/9402284eb91d4b4a897c8528b4ff673c 2024-11-19T17:15:40,013 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/6f5bd7aa18f04a96a3d2179144e79062 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/6f5bd7aa18f04a96a3d2179144e79062 2024-11-19T17:15:40,014 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/8111d3a2913e4ec68b8751732135b856 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/8111d3a2913e4ec68b8751732135b856 2024-11-19T17:15:40,015 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/88c7de26ce264e7183be604e9a7dae30 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/88c7de26ce264e7183be604e9a7dae30 2024-11-19T17:15:40,015 DEBUG [StoreCloser-TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/e83e5ab7f2394f848b3dd9c732c3b757 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/e83e5ab7f2394f848b3dd9c732c3b757 2024-11-19T17:15:40,019 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/recovered.edits/471.seqid, newMaxSeqId=471, maxSeqId=4 2024-11-19T17:15:40,019 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. 2024-11-19T17:15:40,019 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] regionserver.HRegion(1676): Region close journal for 086da227605bf568b877f13b452b7f9a: Waiting for close lock at 1732036537445Running coprocessor pre-close hooks at 1732036537445Disabling compacts and flushes for region at 1732036537445Disabling writes for close at 1732036538315 (+870 ms)Obtaining lock to block concurrent updates at 1732036538315Preparing flush snapshotting stores in 086da227605bf568b877f13b452b7f9a at 1732036538315Finished memstore snapshotting TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732036538315Flushing stores of TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a. at 1732036538316 (+1 ms)Flushing 086da227605bf568b877f13b452b7f9a/A: creating writer at 1732036538316Flushing mob file 086da227605bf568b877f13b452b7f9a/A: appending metadata at 1732036538319 (+3 ms)Flushing mob file 086da227605bf568b877f13b452b7f9a/A: closing flushed file at 1732036538319Flushing 086da227605bf568b877f13b452b7f9a/A: appending metadata at 1732036538727 (+408 ms)Flushing 086da227605bf568b877f13b452b7f9a/A: closing flushed file at 1732036538727Flushing 086da227605bf568b877f13b452b7f9a/B: creating writer at 1732036539134 (+407 ms)Flushing 086da227605bf568b877f13b452b7f9a/B: appending metadata at 1732036539136 (+2 ms)Flushing 086da227605bf568b877f13b452b7f9a/B: closing flushed file at 1732036539136Flushing 086da227605bf568b877f13b452b7f9a/C: creating writer at 1732036539543 (+407 ms)Flushing 086da227605bf568b877f13b452b7f9a/C: appending metadata at 1732036539545 (+2 ms)Flushing 086da227605bf568b877f13b452b7f9a/C: closing flushed file at 1732036539545Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4cfb679a: reopening flushed file at 1732036539951 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24fece4a: reopening flushed file at 1732036539955 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ea9dfdd: reopening flushed file at 1732036539957 (+2 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 086da227605bf568b877f13b452b7f9a in 1646ms, sequenceid=468, compaction requested=true at 1732036539961 (+4 ms)Writing region close event to WAL at 1732036540016 (+55 ms)Running coprocessor post-close hooks at 1732036540019 (+3 ms)Closed at 1732036540019 2024-11-19T17:15:40,021 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=124}] handler.UnassignRegionHandler(157): Closed 086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:40,022 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=086da227605bf568b877f13b452b7f9a, regionState=CLOSED 2024-11-19T17:15:40,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=124, ppid=123, state=RUNNABLE, hasLock=false; CloseRegionProcedure 086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 because future has completed 2024-11-19T17:15:40,025 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-19T17:15:40,025 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; CloseRegionProcedure 086da227605bf568b877f13b452b7f9a, server=280a42d09ec8,33873,1732036422473 in 2.7300 sec 2024-11-19T17:15:40,026 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=123, resume processing ppid=122 2024-11-19T17:15:40,026 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, ppid=122, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=086da227605bf568b877f13b452b7f9a, UNASSIGN in 2.7340 sec 2024-11-19T17:15:40,028 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-19T17:15:40,028 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.7360 sec 2024-11-19T17:15:40,029 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732036540028"}]},"ts":"1732036540028"} 2024-11-19T17:15:40,030 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T17:15:40,030 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T17:15:40,031 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.7450 sec 2024-11-19T17:15:40,709 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T17:15:41,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T17:15:41,444 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:15:41,444 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T17:15:41,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:15:41,446 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=125, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:15:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T17:15:41,447 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=125, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:15:41,448 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:41,450 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A, FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B, FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C, FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/recovered.edits] 2024-11-19T17:15:41,452 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/22eb00ae8bd6420fa01834b29c3b71f8 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/22eb00ae8bd6420fa01834b29c3b71f8 2024-11-19T17:15:41,453 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/9a1aeee6c4ce458a893e9e5379afa444 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/9a1aeee6c4ce458a893e9e5379afa444 2024-11-19T17:15:41,454 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/a121dd95771c43a4a7cd8d7bf2ecd6a8 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/a121dd95771c43a4a7cd8d7bf2ecd6a8 2024-11-19T17:15:41,455 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/dc4b998f5a8e472e9d0a85b5a50d48f7 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/A/dc4b998f5a8e472e9d0a85b5a50d48f7 2024-11-19T17:15:41,457 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/5b9c24cdfa844dcca73cf9a456840f8a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/5b9c24cdfa844dcca73cf9a456840f8a 2024-11-19T17:15:41,457 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/5d54f0f7770c482ab450f9e67bd440c0 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/5d54f0f7770c482ab450f9e67bd440c0 2024-11-19T17:15:41,458 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/653118ddab4848e1b1156122fe803d5c to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/653118ddab4848e1b1156122fe803d5c 2024-11-19T17:15:41,459 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/9846884075324235a7638337bfad8ad3 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/B/9846884075324235a7638337bfad8ad3 2024-11-19T17:15:41,461 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/328ef6571ec8472e8c60423f148d8521 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/328ef6571ec8472e8c60423f148d8521 2024-11-19T17:15:41,462 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/372b3dc7b90f402fbf4a65e83355c97e to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/372b3dc7b90f402fbf4a65e83355c97e 2024-11-19T17:15:41,463 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/6099ec04e6b64b529b251b8fe6a7158f to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/6099ec04e6b64b529b251b8fe6a7158f 2024-11-19T17:15:41,464 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/c5988174daf64eeca2d3c93e1d62796b to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/C/c5988174daf64eeca2d3c93e1d62796b 2024-11-19T17:15:41,466 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/recovered.edits/471.seqid to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a/recovered.edits/471.seqid 2024-11-19T17:15:41,466 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:41,466 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T17:15:41,467 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T17:15:41,467 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T17:15:41,470 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111908d170ccb902444393ee8e8b4b0edda9_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111908d170ccb902444393ee8e8b4b0edda9_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:41,470 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119138864a3dab541b4bc43e6f70bfcd938_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119138864a3dab541b4bc43e6f70bfcd938_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:41,471 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119193f253d6da54b03af985fd91f72c13d_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119193f253d6da54b03af985fd91f72c13d_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:41,472 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119345b2c158b944f499e36c5ab2416ca9a_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119345b2c158b944f499e36c5ab2416ca9a_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:41,473 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111938a15c8d2047488dbd9f5d510ab86e48_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111938a15c8d2047488dbd9f5d510ab86e48_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:41,474 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197269be4816ab40ea99268c932c897b9b_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197269be4816ab40ea99268c932c897b9b_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:41,474 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111985b36af1c0684c43b6c75beb4323ba69_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111985b36af1c0684c43b6c75beb4323ba69_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:41,475 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198887a8dfe1144b179a63311537fe3106_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198887a8dfe1144b179a63311537fe3106_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:41,476 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119976c8d5143a844f8814805b6b04b6c79_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119976c8d5143a844f8814805b6b04b6c79_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:41,476 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a25f7014fb2e4ec289bf6cdc1f6a2a4c_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a25f7014fb2e4ec289bf6cdc1f6a2a4c_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:41,477 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ad6516faa8ac4f1c8c78933d78568696_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ad6516faa8ac4f1c8c78933d78568696_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:41,478 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b2cb8f881dde440b90911487e7d826a5_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b2cb8f881dde440b90911487e7d826a5_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:41,479 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c0b515e804c24839bc34145f35fae8a4_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c0b515e804c24839bc34145f35fae8a4_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:41,479 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c877dfa542624fb4829cd41e1111a71c_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c877dfa542624fb4829cd41e1111a71c_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:41,480 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d113fedc96af4a708fda84e6c5d2dbd9_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d113fedc96af4a708fda84e6c5d2dbd9_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:41,481 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d683399c870140ccb198bb224ad234f5_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d683399c870140ccb198bb224ad234f5_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:41,482 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d87aeb9215c94fb7846b041354e297d3_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d87aeb9215c94fb7846b041354e297d3_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:41,483 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e7be1e51452b4004ad19146e9b4a5cfd_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e7be1e51452b4004ad19146e9b4a5cfd_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:41,484 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f4e647c3be7b4bdb94e48e113cb761cd_086da227605bf568b877f13b452b7f9a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f4e647c3be7b4bdb94e48e113cb761cd_086da227605bf568b877f13b452b7f9a 2024-11-19T17:15:41,484 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T17:15:41,486 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=125, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:15:41,488 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T17:15:41,490 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T17:15:41,491 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=125, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:15:41,491 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T17:15:41,491 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732036541491"}]},"ts":"9223372036854775807"} 2024-11-19T17:15:41,493 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T17:15:41,493 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 086da227605bf568b877f13b452b7f9a, NAME => 'TestAcidGuarantees,,1732036512913.086da227605bf568b877f13b452b7f9a.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T17:15:41,493 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T17:15:41,493 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732036541493"}]},"ts":"9223372036854775807"} 2024-11-19T17:15:41,495 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T17:15:41,495 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=125, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:15:41,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 51 msec 2024-11-19T17:15:41,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T17:15:41,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T17:15:41,554 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:15:41,563 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=251 (was 250) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-873178183_22 at /127.0.0.1:59826 [Waiting for operation #1045] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_569709797_22 at /127.0.0.1:57676 [Waiting for operation #825] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_569709797_22 at /127.0.0.1:57666 [Waiting for operation #838] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-873178183_22 at /127.0.0.1:57650 [Waiting for operation #842] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=380 (was 381), ProcessCount=11 (was 11), AvailableMemoryMB=6160 (was 6673) 2024-11-19T17:15:41,572 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=380, ProcessCount=11, AvailableMemoryMB=6160 2024-11-19T17:15:41,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T17:15:41,573 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T17:15:41,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=126, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T17:15:41,575 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T17:15:41,575 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:41,575 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 126 2024-11-19T17:15:41,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-19T17:15:41,576 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T17:15:41,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742318_1494 (size=1061) 2024-11-19T17:15:41,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-19T17:15:41,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-19T17:15:41,982 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e 2024-11-19T17:15:41,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742319_1495 (size=53) 2024-11-19T17:15:42,132 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T17:15:42,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-19T17:15:42,387 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:15:42,387 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2d49416a7b2a16305b1ea9d2c71b4946, disabling compactions & flushes 2024-11-19T17:15:42,387 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:42,387 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:42,387 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. after waiting 0 ms 2024-11-19T17:15:42,387 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:42,387 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:42,387 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2d49416a7b2a16305b1ea9d2c71b4946: Waiting for close lock at 1732036542387Disabling compacts and flushes for region at 1732036542387Disabling writes for close at 1732036542387Writing region close event to WAL at 1732036542387Closed at 1732036542387 2024-11-19T17:15:42,388 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T17:15:42,388 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732036542388"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732036542388"}]},"ts":"1732036542388"} 2024-11-19T17:15:42,390 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T17:15:42,390 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T17:15:42,390 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732036542390"}]},"ts":"1732036542390"} 2024-11-19T17:15:42,391 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T17:15:42,391 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d49416a7b2a16305b1ea9d2c71b4946, ASSIGN}] 2024-11-19T17:15:42,392 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d49416a7b2a16305b1ea9d2c71b4946, ASSIGN 2024-11-19T17:15:42,393 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d49416a7b2a16305b1ea9d2c71b4946, ASSIGN; state=OFFLINE, location=280a42d09ec8,33873,1732036422473; forceNewPlan=false, retain=false 2024-11-19T17:15:42,543 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=2d49416a7b2a16305b1ea9d2c71b4946, regionState=OPENING, regionLocation=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:42,545 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d49416a7b2a16305b1ea9d2c71b4946, ASSIGN because future has completed 2024-11-19T17:15:42,545 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473}] 2024-11-19T17:15:42,699 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:42,699 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(7752): Opening region: {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:15:42,700 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:15:42,700 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:15:42,700 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(7794): checking encryption for 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:15:42,700 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(7797): checking classloading for 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:15:42,701 INFO [StoreOpener-2d49416a7b2a16305b1ea9d2c71b4946-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:15:42,702 INFO [StoreOpener-2d49416a7b2a16305b1ea9d2c71b4946-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T17:15:42,702 INFO [StoreOpener-2d49416a7b2a16305b1ea9d2c71b4946-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d49416a7b2a16305b1ea9d2c71b4946 columnFamilyName A 2024-11-19T17:15:42,702 DEBUG [StoreOpener-2d49416a7b2a16305b1ea9d2c71b4946-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:42,702 INFO [StoreOpener-2d49416a7b2a16305b1ea9d2c71b4946-1 {}] regionserver.HStore(327): Store=2d49416a7b2a16305b1ea9d2c71b4946/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:15:42,702 INFO [StoreOpener-2d49416a7b2a16305b1ea9d2c71b4946-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:15:42,703 INFO [StoreOpener-2d49416a7b2a16305b1ea9d2c71b4946-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T17:15:42,703 INFO [StoreOpener-2d49416a7b2a16305b1ea9d2c71b4946-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d49416a7b2a16305b1ea9d2c71b4946 columnFamilyName B 2024-11-19T17:15:42,703 DEBUG [StoreOpener-2d49416a7b2a16305b1ea9d2c71b4946-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:42,704 INFO [StoreOpener-2d49416a7b2a16305b1ea9d2c71b4946-1 {}] regionserver.HStore(327): Store=2d49416a7b2a16305b1ea9d2c71b4946/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:15:42,704 INFO [StoreOpener-2d49416a7b2a16305b1ea9d2c71b4946-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:15:42,704 INFO [StoreOpener-2d49416a7b2a16305b1ea9d2c71b4946-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T17:15:42,705 INFO [StoreOpener-2d49416a7b2a16305b1ea9d2c71b4946-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d49416a7b2a16305b1ea9d2c71b4946 columnFamilyName C 2024-11-19T17:15:42,705 DEBUG [StoreOpener-2d49416a7b2a16305b1ea9d2c71b4946-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:15:42,705 INFO [StoreOpener-2d49416a7b2a16305b1ea9d2c71b4946-1 {}] regionserver.HStore(327): Store=2d49416a7b2a16305b1ea9d2c71b4946/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:15:42,705 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:42,705 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1038): replaying wal for 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:15:42,705 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:15:42,706 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:15:42,706 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1048): stopping wal replay for 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:15:42,706 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1060): Cleaning up temporary data for 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:15:42,707 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T17:15:42,707 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1093): writing seq id for 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:15:42,709 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T17:15:42,709 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1114): Opened 2d49416a7b2a16305b1ea9d2c71b4946; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65403301, jitterRate=-0.02541486918926239}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T17:15:42,709 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:15:42,710 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegion(1006): Region open journal for 2d49416a7b2a16305b1ea9d2c71b4946: Running coprocessor pre-open hook at 1732036542700Writing region info on filesystem at 1732036542700Initializing all the Stores at 1732036542700Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036542700Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036542701 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036542701Cleaning up temporary data from old regions at 1732036542706 (+5 ms)Running coprocessor post-open hooks at 1732036542709 (+3 ms)Region opened successfully at 1732036542710 (+1 ms) 2024-11-19T17:15:42,710 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., pid=128, masterSystemTime=1732036542696 2024-11-19T17:15:42,712 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:42,712 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=128}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:42,712 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=2d49416a7b2a16305b1ea9d2c71b4946, regionState=OPEN, openSeqNum=2, regionLocation=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:42,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-19T17:15:42,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=128, ppid=127, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 because future has completed 2024-11-19T17:15:42,715 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-19T17:15:42,715 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; OpenRegionProcedure 2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 in 169 msec 2024-11-19T17:15:42,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=127, resume processing ppid=126 2024-11-19T17:15:42,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, ppid=126, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d49416a7b2a16305b1ea9d2c71b4946, ASSIGN in 324 msec 2024-11-19T17:15:42,717 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T17:15:42,718 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732036542717"}]},"ts":"1732036542717"} 2024-11-19T17:15:42,719 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T17:15:42,719 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=126, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T17:15:42,720 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1460 sec 2024-11-19T17:15:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=126 2024-11-19T17:15:43,724 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:15:43,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6d55bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,725 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:15:43,725 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:15:43,726 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:15:43,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:15:43,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:15:43,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36c13ded, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:15:43,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:15:43,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:43,727 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:15:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7762fa17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,728 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:15:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:15:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:15:43,730 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:15:43,731 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T17:15:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 280a42d09ec8,40603,1732036421715 2024-11-19T17:15:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@639b7332 2024-11-19T17:15:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T17:15:43,733 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T17:15:43,734 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,734 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:15:43,734 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:15:43,735 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:15:43,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:15:43,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:15:43,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20a7da87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:15:43,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:15:43,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:43,736 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:15:43,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a6b2d5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,737 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d7458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,737 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:15:43,737 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:15:43,737 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:15:43,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:15:43,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:15:43,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30e4904f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:15:43,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:15:43,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:43,739 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:15:43,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6514bef8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,740 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c61cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,740 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:15:43,740 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:15:43,740 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:15:43,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:15:43,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:15:43,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75185892, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:15:43,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:15:43,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:43,741 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:15:43,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e95e313, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,742 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,742 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:15:43,742 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:15:43,743 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:15:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:15:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:15:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@502cc841, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:15:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:15:43,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:43,744 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:15:43,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d1d0670, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,745 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47bc1e91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,745 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:15:43,745 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:15:43,746 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:15:43,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:15:43,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:15:43,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7447fe48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:15:43,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:15:43,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:43,747 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:15:43,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@189cb029, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,748 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,748 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:15:43,748 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:15:43,749 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:15:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:15:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:15:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@546d2b65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:15:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:15:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:43,750 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:15:43,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16f03c42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,751 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,751 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:15:43,751 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:15:43,751 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:15:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:15:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:15:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49eaefde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:15:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:15:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:43,753 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:15:43,754 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b245127, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,754 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,754 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:15:43,754 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:15:43,755 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:15:43,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:15:43,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:15:43,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3df82f08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:15:43,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:15:43,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:43,756 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:15:43,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26c1a729, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,757 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,757 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:15:43,757 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:15:43,758 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:15:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:15:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:15:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6146daca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:15:43,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:15:43,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:43,759 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:15:43,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0817a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f5639f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,760 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:15:43,760 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:15:43,761 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:15:43,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:15:43,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:15:43,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55080f9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:15:43,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:15:43,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:15:43,762 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:15:43,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8cf0755, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:15:43,763 DEBUG [Thread-2196 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:15:43,764 DEBUG [Thread-2211 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:15:43,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:15:43,764 DEBUG [Thread-2200 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:15:43,764 DEBUG [Thread-2207 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:15:43,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:15:43,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:15:43,765 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:15:43,765 DEBUG [Thread-2213 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:15:43,765 DEBUG [Thread-2198 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:15:43,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:15:43,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:15:43,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:15:43,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:15:43,765 DEBUG [Thread-2215 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:15:43,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:15:43,765 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:15:43,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:15:43,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:15:43,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:15:43,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:15:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-19T17:15:43,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:15:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:15:43,767 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:15:43,767 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:15:43,767 DEBUG [Thread-2202 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:15:43,767 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:15:43,767 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:15:43,767 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:15:43,767 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:15:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T17:15:43,768 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:15:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:15:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:15:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2] 2024-11-19T17:15:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2] 2024-11-19T17:15:43,769 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:15:43,769 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:15:43,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2] 2024-11-19T17:15:43,770 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:15:43,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2] 2024-11-19T17:15:43,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2] 2024-11-19T17:15:43,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2] 2024-11-19T17:15:43,771 DEBUG [Thread-2209 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:15:43,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2] 2024-11-19T17:15:43,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:15:43,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2] 2024-11-19T17:15:43,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:15:43,773 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:15:43,773 DEBUG [Thread-2204 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:15:43,774 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2] 2024-11-19T17:15:43,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:15:43,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:15:43,777 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:15:43,778 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2] 2024-11-19T17:15:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:15:43,781 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d49416a7b2a16305b1ea9d2c71b4946 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T17:15:43,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=A 2024-11-19T17:15:43,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:43,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=B 2024-11-19T17:15:43,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:43,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=C 2024-11-19T17:15:43,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:43,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/e39149d3b4a34687946f4eb6079a35f8 is 50, key is test_row_0/A:col10/1732036543779/Put/seqid=0 2024-11-19T17:15:43,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742320_1496 (size=16681) 2024-11-19T17:15:43,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/e39149d3b4a34687946f4eb6079a35f8 2024-11-19T17:15:43,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036543818, max timestamp=1732036543836Num uniques 859; 2024-11-19T17:15:43,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036543818, max timestamp=1732036543836Num uniques 859; 2024-11-19T17:15:43,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732036543813, max timestamp=1732036543836Num uniques 859; 2024-11-19T17:15:43,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:15:43,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:15:43,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T17:15:43,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/933281e693c64ecf9209f25cc1721947 is 50, key is test_row_0/B:col10/1732036543779/Put/seqid=0 2024-11-19T17:15:43,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742321_1497 (size=12001) 2024-11-19T17:15:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T17:15:43,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036603870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:43,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:43,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51578 deadline: 1732036603872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:43,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:43,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:43,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:43,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036603882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:43,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:43,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:43,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:43,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:43,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:43,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51512 deadline: 1732036603883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:43,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:43,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:43,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036603884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:43,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:43,897 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:43,897 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:43,897 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:43,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T17:15:43,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:43,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:43,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:43,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:43,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:44,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:44,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036603994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:44,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51578 deadline: 1732036603994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:44,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036603994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:44,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:44,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51512 deadline: 1732036604003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:44,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:44,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:44,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:44,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:44,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036604003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:44,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:44,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:44,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:44,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T17:15:44,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:44,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:44,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:44,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:44,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:44,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T17:15:44,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:44,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036604213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:44,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:44,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:44,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51578 deadline: 1732036604213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:44,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:44,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:44,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:44,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:44,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036604213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:44,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:44,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:44,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:44,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:44,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51512 deadline: 1732036604214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:44,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:44,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:44,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:44,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:44,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036604224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:44,226 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:44,226 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:44,226 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:44,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T17:15:44,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:44,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:44,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:44,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:44,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:44,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:44,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/933281e693c64ecf9209f25cc1721947 2024-11-19T17:15:44,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/bc4f2baf30734bc4a4b7d28849d5f68f is 50, key is test_row_0/C:col10/1732036543779/Put/seqid=0 2024-11-19T17:15:44,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742322_1498 (size=12001) 2024-11-19T17:15:44,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T17:15:44,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:44,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:44,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:44,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:44,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T17:15:44,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T17:15:44,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:44,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:44,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:44,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:44,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:44,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:44,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:44,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036604533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:44,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:44,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:44,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51578 deadline: 1732036604533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:44,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:44,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:44,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:44,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:44,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036604533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:44,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:44,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:44,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:44,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:44,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:44,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:44,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51512 deadline: 1732036604533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:44,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:44,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:44,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:44,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:44,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036604534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:44,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:44,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:44,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:44,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/bc4f2baf30734bc4a4b7d28849d5f68f 2024-11-19T17:15:44,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T17:15:44,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:44,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:44,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:44,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:44,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:44,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/e39149d3b4a34687946f4eb6079a35f8 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/e39149d3b4a34687946f4eb6079a35f8 2024-11-19T17:15:44,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:44,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/e39149d3b4a34687946f4eb6079a35f8, entries=250, sequenceid=12, filesize=16.3 K 2024-11-19T17:15:44,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/933281e693c64ecf9209f25cc1721947 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/933281e693c64ecf9209f25cc1721947 2024-11-19T17:15:44,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/933281e693c64ecf9209f25cc1721947, entries=150, sequenceid=12, filesize=11.7 K 2024-11-19T17:15:44,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/bc4f2baf30734bc4a4b7d28849d5f68f as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/bc4f2baf30734bc4a4b7d28849d5f68f 2024-11-19T17:15:44,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/bc4f2baf30734bc4a4b7d28849d5f68f, entries=150, sequenceid=12, filesize=11.7 K 2024-11-19T17:15:44,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.71 KB/132820 for 2d49416a7b2a16305b1ea9d2c71b4946 in 920ms, sequenceid=12, compaction requested=false 2024-11-19T17:15:44,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:44,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T17:15:44,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:44,843 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 2d49416a7b2a16305b1ea9d2c71b4946 3/3 column families, dataSize=129.71 KB heapSize=365.45 KB 2024-11-19T17:15:44,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=A 2024-11-19T17:15:44,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:44,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=B 2024-11-19T17:15:44,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:44,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=C 2024-11-19T17:15:44,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:44,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/95e85861fa7041d0b5569eac7a900f29 is 50, key is test_row_0/A:col10/1732036543881/Put/seqid=0 2024-11-19T17:15:44,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742323_1499 (size=12001) 2024-11-19T17:15:44,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T17:15:45,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:15:45,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:45,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:45,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51512 deadline: 1732036605064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:45,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036605065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:45,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036605067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:45,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:45,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51578 deadline: 1732036605077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:45,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:45,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:45,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036605078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:45,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:45,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:45,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51512 deadline: 1732036605184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:45,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:45,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036605184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:45,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:45,194 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036605184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:45,194 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,194 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:45,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:45,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51578 deadline: 1732036605184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:45,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:45,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:45,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:45,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036605194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:45,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:45,254 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/95e85861fa7041d0b5569eac7a900f29 2024-11-19T17:15:45,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/9c1c5a5233ee49cf8b41a84798dc44d2 is 50, key is test_row_0/B:col10/1732036543881/Put/seqid=0 2024-11-19T17:15:45,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742324_1500 (size=12001) 2024-11-19T17:15:45,278 WARN [regionserver/280a42d09ec8:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 5, running: 1 2024-11-19T17:15:45,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:45,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51512 deadline: 1732036605403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:45,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:45,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:45,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036605403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:45,413 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:45,413 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,413 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:45,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036605403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:45,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:45,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:45,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51578 deadline: 1732036605404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:45,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:45,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:45,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036605404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:45,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:45,541 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T17:15:45,666 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/9c1c5a5233ee49cf8b41a84798dc44d2 2024-11-19T17:15:45,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/e08d583706fb44d1829d547ab901c005 is 50, key is test_row_0/C:col10/1732036543881/Put/seqid=0 2024-11-19T17:15:45,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742325_1501 (size=12001) 2024-11-19T17:15:45,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:45,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51512 deadline: 1732036605723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:45,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:45,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036605724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:45,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:45,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:45,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036605724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:45,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:45,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:45,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:45,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51578 deadline: 1732036605724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:45,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:45,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036605724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T17:15:46,077 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/e08d583706fb44d1829d547ab901c005 2024-11-19T17:15:46,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/95e85861fa7041d0b5569eac7a900f29 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/95e85861fa7041d0b5569eac7a900f29 2024-11-19T17:15:46,084 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/95e85861fa7041d0b5569eac7a900f29, entries=150, sequenceid=47, filesize=11.7 K 2024-11-19T17:15:46,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/9c1c5a5233ee49cf8b41a84798dc44d2 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/9c1c5a5233ee49cf8b41a84798dc44d2 2024-11-19T17:15:46,088 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/9c1c5a5233ee49cf8b41a84798dc44d2, entries=150, sequenceid=47, filesize=11.7 K 2024-11-19T17:15:46,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/e08d583706fb44d1829d547ab901c005 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/e08d583706fb44d1829d547ab901c005 2024-11-19T17:15:46,092 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/e08d583706fb44d1829d547ab901c005, entries=150, sequenceid=47, filesize=11.7 K 2024-11-19T17:15:46,092 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~365.41 KB/374176, currentSize=53.67 KB/54960 for 2d49416a7b2a16305b1ea9d2c71b4946 in 1250ms, sequenceid=47, compaction requested=false 2024-11-19T17:15:46,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:46,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:46,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-19T17:15:46,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-19T17:15:46,095 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-19T17:15:46,095 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3250 sec 2024-11-19T17:15:46,097 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.3310 sec 2024-11-19T17:15:46,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:15:46,240 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d49416a7b2a16305b1ea9d2c71b4946 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T17:15:46,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=A 2024-11-19T17:15:46,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:46,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=B 2024-11-19T17:15:46,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:46,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=C 2024-11-19T17:15:46,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:46,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/fe89bc1d82da44c7a7ee4a92e00ed5f5 is 50, key is test_row_0/A:col10/1732036546239/Put/seqid=0 2024-11-19T17:15:46,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742326_1502 (size=14341) 2024-11-19T17:15:46,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036606313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:46,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036606314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:46,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:46,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036606316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:46,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:46,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732036546311, max timestamp=1732036546326Num uniques 859; 2024-11-19T17:15:46,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732036546311, max timestamp=1732036546326Num uniques 859; 2024-11-19T17:15:46,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:15:46,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:15:46,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:46,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732036546311, max timestamp=1732036546326Num uniques 859; 2024-11-19T17:15:46,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:15:46,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:46,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51512 deadline: 1732036606406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:46,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:46,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51578 deadline: 1732036606418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:46,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:46,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036606434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:46,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036606434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:46,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036606434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:46,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:46,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51512 deadline: 1732036606534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:46,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:46,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:46,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51578 deadline: 1732036606544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:46,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:46,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/fe89bc1d82da44c7a7ee4a92e00ed5f5 2024-11-19T17:15:46,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/63f68dc15f554741bd9751ec5a976de1 is 50, key is test_row_0/B:col10/1732036546239/Put/seqid=0 2024-11-19T17:15:46,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036606654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:46,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036606654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:46,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,667 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:46,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036606654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:46,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:46,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:46,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742327_1503 (size=12001) 2024-11-19T17:15:46,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51512 deadline: 1732036606753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:46,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:46,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51578 deadline: 1732036606763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:46,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:46,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:46,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036606973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:46,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:46,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036606973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:46,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:46,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:46,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036606973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:46,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:46,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:46,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:47,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/63f68dc15f554741bd9751ec5a976de1 2024-11-19T17:15:47,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:47,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51512 deadline: 1732036607073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:47,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:47,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:47,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:47,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/287798b5c69f4bf99c3dc9418c9e2bfd is 50, key is test_row_0/C:col10/1732036546239/Put/seqid=0 2024-11-19T17:15:47,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742328_1504 (size=12001) 2024-11-19T17:15:47,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:47,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51578 deadline: 1732036607083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:47,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:47,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:47,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:47,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/287798b5c69f4bf99c3dc9418c9e2bfd 2024-11-19T17:15:47,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/fe89bc1d82da44c7a7ee4a92e00ed5f5 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/fe89bc1d82da44c7a7ee4a92e00ed5f5 2024-11-19T17:15:47,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/fe89bc1d82da44c7a7ee4a92e00ed5f5, entries=200, sequenceid=59, filesize=14.0 K 2024-11-19T17:15:47,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/63f68dc15f554741bd9751ec5a976de1 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/63f68dc15f554741bd9751ec5a976de1 2024-11-19T17:15:47,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/63f68dc15f554741bd9751ec5a976de1, entries=150, sequenceid=59, filesize=11.7 K 2024-11-19T17:15:47,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/287798b5c69f4bf99c3dc9418c9e2bfd as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/287798b5c69f4bf99c3dc9418c9e2bfd 2024-11-19T17:15:47,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/287798b5c69f4bf99c3dc9418c9e2bfd, entries=150, sequenceid=59, filesize=11.7 K 2024-11-19T17:15:47,504 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-19T17:15:47,505 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 2d49416a7b2a16305b1ea9d2c71b4946 in 1265ms, sequenceid=59, compaction requested=true 2024-11-19T17:15:47,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:47,505 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:47,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d49416a7b2a16305b1ea9d2c71b4946:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:15:47,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:47,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d49416a7b2a16305b1ea9d2c71b4946:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:15:47,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:47,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d49416a7b2a16305b1ea9d2c71b4946:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:15:47,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:47,506 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:47,506 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:47,506 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 2d49416a7b2a16305b1ea9d2c71b4946/A is initiating minor compaction (all files) 2024-11-19T17:15:47,507 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d49416a7b2a16305b1ea9d2c71b4946/A in TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:47,507 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/e39149d3b4a34687946f4eb6079a35f8, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/95e85861fa7041d0b5569eac7a900f29, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/fe89bc1d82da44c7a7ee4a92e00ed5f5] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp, totalSize=42.0 K 2024-11-19T17:15:47,507 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:47,507 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 2d49416a7b2a16305b1ea9d2c71b4946/B is initiating minor compaction (all files) 2024-11-19T17:15:47,507 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d49416a7b2a16305b1ea9d2c71b4946/B in TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:47,507 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/933281e693c64ecf9209f25cc1721947, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/9c1c5a5233ee49cf8b41a84798dc44d2, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/63f68dc15f554741bd9751ec5a976de1] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp, totalSize=35.2 K 2024-11-19T17:15:47,508 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 933281e693c64ecf9209f25cc1721947, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732036543774 2024-11-19T17:15:47,508 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting e39149d3b4a34687946f4eb6079a35f8, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732036543774 2024-11-19T17:15:47,508 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c1c5a5233ee49cf8b41a84798dc44d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732036543866 2024-11-19T17:15:47,508 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95e85861fa7041d0b5569eac7a900f29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732036543866 2024-11-19T17:15:47,508 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 63f68dc15f554741bd9751ec5a976de1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732036545065 2024-11-19T17:15:47,508 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe89bc1d82da44c7a7ee4a92e00ed5f5, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732036545065 2024-11-19T17:15:47,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:15:47,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d49416a7b2a16305b1ea9d2c71b4946 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-19T17:15:47,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=A 2024-11-19T17:15:47,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:47,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=B 2024-11-19T17:15:47,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:47,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=C 2024-11-19T17:15:47,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:47,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/2141df3479a74e0c91df5bfa77a4a136 is 50, key is test_row_0/A:col10/1732036546404/Put/seqid=0 2024-11-19T17:15:47,516 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d49416a7b2a16305b1ea9d2c71b4946#B#compaction#422 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:47,517 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d49416a7b2a16305b1ea9d2c71b4946#A#compaction#423 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:47,517 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/0bfb4432c0d14e7299cc9c75f9483d85 is 50, key is test_row_0/B:col10/1732036546239/Put/seqid=0 2024-11-19T17:15:47,517 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/09fde3fb2a63424a890f02f82f57938e is 50, key is test_row_0/A:col10/1732036546239/Put/seqid=0 2024-11-19T17:15:47,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742329_1505 (size=14341) 2024-11-19T17:15:47,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742331_1507 (size=12104) 2024-11-19T17:15:47,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742330_1506 (size=12104) 2024-11-19T17:15:47,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036607533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:47,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036607536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:47,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036607551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:47,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:47,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:47,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:47,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51512 deadline: 1732036607594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:47,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:47,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:47,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:47,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51578 deadline: 1732036607603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:47,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:47,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:47,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:47,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036607654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:47,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:47,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036607654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:47,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:47,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:47,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:47,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:47,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:47,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036607673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:47,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:47,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:47,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:47,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:47,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036607874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:47,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:47,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036607874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:47,886 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:47,886 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:47,886 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:47,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:47,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:47,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:47,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:47,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036607893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:47,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:47,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:47,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:47,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T17:15:47,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/2141df3479a74e0c91df5bfa77a4a136 2024-11-19T17:15:47,924 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:15:47,924 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:15:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-19T17:15:47,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T17:15:47,926 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:15:47,927 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:15:47,927 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:15:47,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/808f107e297741f695d2b68efcdc50de is 50, key is test_row_0/B:col10/1732036546404/Put/seqid=0 2024-11-19T17:15:47,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742332_1508 (size=12001) 2024-11-19T17:15:47,940 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/09fde3fb2a63424a890f02f82f57938e as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/09fde3fb2a63424a890f02f82f57938e 2024-11-19T17:15:47,940 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/0bfb4432c0d14e7299cc9c75f9483d85 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/0bfb4432c0d14e7299cc9c75f9483d85 2024-11-19T17:15:47,944 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d49416a7b2a16305b1ea9d2c71b4946/B of 2d49416a7b2a16305b1ea9d2c71b4946 into 0bfb4432c0d14e7299cc9c75f9483d85(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:47,944 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d49416a7b2a16305b1ea9d2c71b4946/A of 2d49416a7b2a16305b1ea9d2c71b4946 into 09fde3fb2a63424a890f02f82f57938e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:47,944 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:47,944 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:47,944 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., storeName=2d49416a7b2a16305b1ea9d2c71b4946/B, priority=13, startTime=1732036547506; duration=0sec 2024-11-19T17:15:47,944 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., storeName=2d49416a7b2a16305b1ea9d2c71b4946/A, priority=13, startTime=1732036547505; duration=0sec 2024-11-19T17:15:47,944 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:47,944 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d49416a7b2a16305b1ea9d2c71b4946:B 2024-11-19T17:15:47,944 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:47,944 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d49416a7b2a16305b1ea9d2c71b4946:A 2024-11-19T17:15:47,944 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:47,945 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:47,945 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 2d49416a7b2a16305b1ea9d2c71b4946/C is initiating minor compaction (all files) 2024-11-19T17:15:47,945 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d49416a7b2a16305b1ea9d2c71b4946/C in TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:47,945 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/bc4f2baf30734bc4a4b7d28849d5f68f, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/e08d583706fb44d1829d547ab901c005, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/287798b5c69f4bf99c3dc9418c9e2bfd] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp, totalSize=35.2 K 2024-11-19T17:15:47,946 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting bc4f2baf30734bc4a4b7d28849d5f68f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732036543774 2024-11-19T17:15:47,946 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting e08d583706fb44d1829d547ab901c005, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732036543866 2024-11-19T17:15:47,947 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 287798b5c69f4bf99c3dc9418c9e2bfd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732036545065 2024-11-19T17:15:47,957 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d49416a7b2a16305b1ea9d2c71b4946#C#compaction#425 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:47,957 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/956c1dc238604c03bca6f5a067da8aad is 50, key is test_row_0/C:col10/1732036546239/Put/seqid=0 2024-11-19T17:15:47,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742333_1509 (size=12104) 2024-11-19T17:15:48,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T17:15:48,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T17:15:48,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:48,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:48,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:48,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:48,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:48,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:48,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036608193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:48,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036608194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:48,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:48,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036608214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:48,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:48,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T17:15:48,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:48,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:48,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:48,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:48,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:48,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T17:15:48,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/808f107e297741f695d2b68efcdc50de 2024-11-19T17:15:48,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/32ef8153b7734b5fa808aad16fe230c3 is 50, key is test_row_0/C:col10/1732036546404/Put/seqid=0 2024-11-19T17:15:48,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742334_1510 (size=12001) 2024-11-19T17:15:48,365 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/956c1dc238604c03bca6f5a067da8aad as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/956c1dc238604c03bca6f5a067da8aad 2024-11-19T17:15:48,370 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d49416a7b2a16305b1ea9d2c71b4946/C of 2d49416a7b2a16305b1ea9d2c71b4946 into 956c1dc238604c03bca6f5a067da8aad(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:48,370 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:48,370 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., storeName=2d49416a7b2a16305b1ea9d2c71b4946/C, priority=13, startTime=1732036547506; duration=0sec 2024-11-19T17:15:48,370 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:48,370 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d49416a7b2a16305b1ea9d2c71b4946:C 2024-11-19T17:15:48,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T17:15:48,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:48,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:48,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:48,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:48,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:48,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:48,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T17:15:48,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:48,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:48,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:48,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:48,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:48,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:48,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T17:15:48,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:48,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51512 deadline: 1732036608614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:48,619 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_1' in 2d49416a7b2a16305b1ea9d2c71b4946 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:15:48,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:48,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:48,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:48,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:48,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51578 deadline: 1732036608633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:48,643 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_2' in 2d49416a7b2a16305b1ea9d2c71b4946 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:15:48,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:48,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:48,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:48,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T17:15:48,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:48,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:48,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:48,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:48,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:48,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:48,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:48,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036608704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:48,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:48,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:48,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:48,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:48,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036608713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:48,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:48,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036608734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:48,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:48,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:48,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:48,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/32ef8153b7734b5fa808aad16fe230c3 2024-11-19T17:15:48,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/2141df3479a74e0c91df5bfa77a4a136 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/2141df3479a74e0c91df5bfa77a4a136 2024-11-19T17:15:48,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/2141df3479a74e0c91df5bfa77a4a136, entries=200, sequenceid=94, filesize=14.0 K 2024-11-19T17:15:48,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/808f107e297741f695d2b68efcdc50de as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/808f107e297741f695d2b68efcdc50de 2024-11-19T17:15:48,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/808f107e297741f695d2b68efcdc50de, entries=150, sequenceid=94, filesize=11.7 K 2024-11-19T17:15:48,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/32ef8153b7734b5fa808aad16fe230c3 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/32ef8153b7734b5fa808aad16fe230c3 2024-11-19T17:15:48,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/32ef8153b7734b5fa808aad16fe230c3, entries=150, sequenceid=94, filesize=11.7 K 2024-11-19T17:15:48,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 2d49416a7b2a16305b1ea9d2c71b4946 in 1266ms, sequenceid=94, compaction requested=false 2024-11-19T17:15:48,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:48,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T17:15:48,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:48,847 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 2d49416a7b2a16305b1ea9d2c71b4946 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T17:15:48,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=A 2024-11-19T17:15:48,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:48,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=B 2024-11-19T17:15:48,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:48,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=C 2024-11-19T17:15:48,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:48,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/e3bae5df23c34a6d8b86e75998b438dc is 50, key is test_row_0/A:col10/1732036547534/Put/seqid=0 2024-11-19T17:15:48,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742335_1511 (size=12001) 2024-11-19T17:15:49,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T17:15:49,256 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/e3bae5df23c34a6d8b86e75998b438dc 2024-11-19T17:15:49,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/800c5a6a025241e7955f9286cf83d4b3 is 50, key is test_row_0/B:col10/1732036547534/Put/seqid=0 2024-11-19T17:15:49,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742336_1512 (size=12001) 2024-11-19T17:15:49,669 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/800c5a6a025241e7955f9286cf83d4b3 2024-11-19T17:15:49,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/39fa028e6b6c42c5988717438aff339e is 50, key is test_row_0/C:col10/1732036547534/Put/seqid=0 2024-11-19T17:15:49,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742337_1513 (size=12001) 2024-11-19T17:15:49,721 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/39fa028e6b6c42c5988717438aff339e 2024-11-19T17:15:49,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:15:49,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/e3bae5df23c34a6d8b86e75998b438dc as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/e3bae5df23c34a6d8b86e75998b438dc 2024-11-19T17:15:49,748 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/e3bae5df23c34a6d8b86e75998b438dc, entries=150, sequenceid=108, filesize=11.7 K 2024-11-19T17:15:49,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/800c5a6a025241e7955f9286cf83d4b3 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/800c5a6a025241e7955f9286cf83d4b3 2024-11-19T17:15:49,756 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/800c5a6a025241e7955f9286cf83d4b3, entries=150, sequenceid=108, filesize=11.7 K 2024-11-19T17:15:49,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/39fa028e6b6c42c5988717438aff339e as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/39fa028e6b6c42c5988717438aff339e 2024-11-19T17:15:49,765 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/39fa028e6b6c42c5988717438aff339e, entries=150, sequenceid=108, filesize=11.7 K 2024-11-19T17:15:49,766 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=46.96 KB/48090 for 2d49416a7b2a16305b1ea9d2c71b4946 in 919ms, sequenceid=108, compaction requested=true 2024-11-19T17:15:49,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:49,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:49,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-19T17:15:49,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-19T17:15:49,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d49416a7b2a16305b1ea9d2c71b4946 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T17:15:49,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=A 2024-11-19T17:15:49,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:49,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=B 2024-11-19T17:15:49,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:49,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=C 2024-11-19T17:15:49,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:49,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:15:49,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-19T17:15:49,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8410 sec 2024-11-19T17:15:49,775 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.8480 sec 2024-11-19T17:15:49,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/9951c95ca7064ebea6a5fe96d8d02779 is 50, key is test_row_0/A:col10/1732036549760/Put/seqid=0 2024-11-19T17:15:49,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742338_1514 (size=19021) 2024-11-19T17:15:49,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/9951c95ca7064ebea6a5fe96d8d02779 2024-11-19T17:15:49,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/a1bba2a1ff4d4d929f5f5dc9e08497a8 is 50, key is test_row_0/B:col10/1732036549760/Put/seqid=0 2024-11-19T17:15:49,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742339_1515 (size=12001) 2024-11-19T17:15:49,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/a1bba2a1ff4d4d929f5f5dc9e08497a8 2024-11-19T17:15:49,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/577fd51d3e8044b0b1ed0fd09e0ead7f is 50, key is test_row_0/C:col10/1732036549760/Put/seqid=0 2024-11-19T17:15:49,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732036549824, max timestamp=1732036549864Num uniques 859; 2024-11-19T17:15:49,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T17:15:49,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732036549824, max timestamp=1732036549864Num uniques 859; 2024-11-19T17:15:49,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T17:15:49,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732036549824, max timestamp=1732036549864Num uniques 859; 2024-11-19T17:15:49,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T17:15:49,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742340_1516 (size=12001) 2024-11-19T17:15:49,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:49,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036609923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:49,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:49,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:49,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:49,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036609939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:49,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:49,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:49,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:49,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:49,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036609941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:49,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:49,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:49,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:50,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:50,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036610044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:50,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:50,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:50,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:50,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:50,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036610044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:50,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:50,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:50,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:50,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T17:15:50,074 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:15:50,074 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:15:50,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-19T17:15:50,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T17:15:50,076 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:15:50,077 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:15:50,077 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:15:50,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:50,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036610063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:50,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:50,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:50,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:50,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T17:15:50,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-19T17:15:50,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:50,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:50,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:50,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:50,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:50,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:50,273 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/577fd51d3e8044b0b1ed0fd09e0ead7f 2024-11-19T17:15:50,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:50,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036610264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:50,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:50,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:50,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036610264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:50,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:50,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:50,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:50,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:50,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:50,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/9951c95ca7064ebea6a5fe96d8d02779 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/9951c95ca7064ebea6a5fe96d8d02779 2024-11-19T17:15:50,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/9951c95ca7064ebea6a5fe96d8d02779, entries=300, sequenceid=119, filesize=18.6 K 2024-11-19T17:15:50,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/a1bba2a1ff4d4d929f5f5dc9e08497a8 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/a1bba2a1ff4d4d929f5f5dc9e08497a8 2024-11-19T17:15:50,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/a1bba2a1ff4d4d929f5f5dc9e08497a8, entries=150, sequenceid=119, filesize=11.7 K 2024-11-19T17:15:50,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/577fd51d3e8044b0b1ed0fd09e0ead7f as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/577fd51d3e8044b0b1ed0fd09e0ead7f 2024-11-19T17:15:50,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/577fd51d3e8044b0b1ed0fd09e0ead7f, entries=150, sequenceid=119, filesize=11.7 K 2024-11-19T17:15:50,288 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 2d49416a7b2a16305b1ea9d2c71b4946 in 521ms, sequenceid=119, compaction requested=true 2024-11-19T17:15:50,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:50,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d49416a7b2a16305b1ea9d2c71b4946:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:15:50,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:50,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d49416a7b2a16305b1ea9d2c71b4946:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:15:50,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:50,288 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:15:50,288 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:15:50,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d49416a7b2a16305b1ea9d2c71b4946:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:15:50,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:50,290 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57467 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:15:50,290 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:15:50,290 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 2d49416a7b2a16305b1ea9d2c71b4946/A is initiating minor compaction (all files) 2024-11-19T17:15:50,290 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 2d49416a7b2a16305b1ea9d2c71b4946/B is initiating minor compaction (all files) 2024-11-19T17:15:50,290 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d49416a7b2a16305b1ea9d2c71b4946/A in TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:50,290 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d49416a7b2a16305b1ea9d2c71b4946/B in TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:50,290 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/09fde3fb2a63424a890f02f82f57938e, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/2141df3479a74e0c91df5bfa77a4a136, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/e3bae5df23c34a6d8b86e75998b438dc, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/9951c95ca7064ebea6a5fe96d8d02779] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp, totalSize=56.1 K 2024-11-19T17:15:50,290 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/0bfb4432c0d14e7299cc9c75f9483d85, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/808f107e297741f695d2b68efcdc50de, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/800c5a6a025241e7955f9286cf83d4b3, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/a1bba2a1ff4d4d929f5f5dc9e08497a8] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp, totalSize=47.0 K 2024-11-19T17:15:50,290 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09fde3fb2a63424a890f02f82f57938e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732036545065 2024-11-19T17:15:50,290 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bfb4432c0d14e7299cc9c75f9483d85, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732036545065 2024-11-19T17:15:50,290 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 808f107e297741f695d2b68efcdc50de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732036546404 2024-11-19T17:15:50,290 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2141df3479a74e0c91df5bfa77a4a136, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732036546361 2024-11-19T17:15:50,291 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 800c5a6a025241e7955f9286cf83d4b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732036547527 2024-11-19T17:15:50,291 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3bae5df23c34a6d8b86e75998b438dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732036547527 2024-11-19T17:15:50,291 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting a1bba2a1ff4d4d929f5f5dc9e08497a8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732036549752 2024-11-19T17:15:50,291 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9951c95ca7064ebea6a5fe96d8d02779, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732036549742 2024-11-19T17:15:50,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:15:50,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d49416a7b2a16305b1ea9d2c71b4946 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-19T17:15:50,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=A 2024-11-19T17:15:50,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:50,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=B 2024-11-19T17:15:50,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:50,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=C 2024-11-19T17:15:50,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:50,301 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d49416a7b2a16305b1ea9d2c71b4946#B#compaction#433 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:50,301 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/3d00664c7dfa49ce9357c7910ce42fa4 is 50, key is test_row_0/B:col10/1732036549760/Put/seqid=0 2024-11-19T17:15:50,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/5b004a0dad354150b6c9d9ae886f5a98 is 50, key is test_row_0/A:col10/1732036549939/Put/seqid=0 2024-11-19T17:15:50,304 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d49416a7b2a16305b1ea9d2c71b4946#A#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:50,305 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/36257bb45aaf4632be48d9a543639a37 is 50, key is test_row_0/A:col10/1732036549760/Put/seqid=0 2024-11-19T17:15:50,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742341_1517 (size=12241) 2024-11-19T17:15:50,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742342_1518 (size=14541) 2024-11-19T17:15:50,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742343_1519 (size=12241) 2024-11-19T17:15:50,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-19T17:15:50,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:50,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:50,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:50,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:50,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:50,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:50,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T17:15:50,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:50,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036610390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:50,411 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:50,411 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:50,411 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:50,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:50,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036610513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:50,524 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:50,524 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:50,524 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:50,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-19T17:15:50,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:50,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:50,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:50,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:50,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:50,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:50,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:50,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036610583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:50,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:50,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:50,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:50,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:50,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036610583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:50,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:50,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:50,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:50,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:50,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51512 deadline: 1732036610634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:50,646 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_1' in 2d49416a7b2a16305b1ea9d2c71b4946 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:15:50,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:50,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:50,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:50,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:50,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51578 deadline: 1732036610653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:50,662 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_2' in 2d49416a7b2a16305b1ea9d2c71b4946 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:15:50,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:50,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:50,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:50,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-19T17:15:50,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:50,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:50,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:50,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:50,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:50,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T17:15:50,715 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/3d00664c7dfa49ce9357c7910ce42fa4 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/3d00664c7dfa49ce9357c7910ce42fa4 2024-11-19T17:15:50,719 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2d49416a7b2a16305b1ea9d2c71b4946/B of 2d49416a7b2a16305b1ea9d2c71b4946 into 3d00664c7dfa49ce9357c7910ce42fa4(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:50,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/5b004a0dad354150b6c9d9ae886f5a98 2024-11-19T17:15:50,719 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:50,719 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., storeName=2d49416a7b2a16305b1ea9d2c71b4946/B, priority=12, startTime=1732036550288; duration=0sec 2024-11-19T17:15:50,719 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:50,719 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d49416a7b2a16305b1ea9d2c71b4946:B 2024-11-19T17:15:50,719 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:15:50,720 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:15:50,720 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 2d49416a7b2a16305b1ea9d2c71b4946/C is initiating minor compaction (all files) 2024-11-19T17:15:50,721 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d49416a7b2a16305b1ea9d2c71b4946/C in TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:50,721 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/956c1dc238604c03bca6f5a067da8aad, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/32ef8153b7734b5fa808aad16fe230c3, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/39fa028e6b6c42c5988717438aff339e, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/577fd51d3e8044b0b1ed0fd09e0ead7f] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp, totalSize=47.0 K 2024-11-19T17:15:50,722 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 956c1dc238604c03bca6f5a067da8aad, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732036545065 2024-11-19T17:15:50,722 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 32ef8153b7734b5fa808aad16fe230c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732036546404 2024-11-19T17:15:50,723 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 39fa028e6b6c42c5988717438aff339e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732036547527 2024-11-19T17:15:50,724 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 577fd51d3e8044b0b1ed0fd09e0ead7f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732036549752 2024-11-19T17:15:50,726 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/36257bb45aaf4632be48d9a543639a37 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/36257bb45aaf4632be48d9a543639a37 2024-11-19T17:15:50,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/d1114a6dcce44287899832b7e2e94931 is 50, key is test_row_0/B:col10/1732036549939/Put/seqid=0 2024-11-19T17:15:50,731 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2d49416a7b2a16305b1ea9d2c71b4946/A of 2d49416a7b2a16305b1ea9d2c71b4946 into 36257bb45aaf4632be48d9a543639a37(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:50,731 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:50,731 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., storeName=2d49416a7b2a16305b1ea9d2c71b4946/A, priority=12, startTime=1732036550288; duration=0sec 2024-11-19T17:15:50,731 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:50,731 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d49416a7b2a16305b1ea9d2c71b4946:A 2024-11-19T17:15:50,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742344_1520 (size=12151) 2024-11-19T17:15:50,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/d1114a6dcce44287899832b7e2e94931 2024-11-19T17:15:50,736 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d49416a7b2a16305b1ea9d2c71b4946#C#compaction#437 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:50,737 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/adf4cd39e7b0475c9495d9afe2b76fa4 is 50, key is test_row_0/C:col10/1732036549760/Put/seqid=0 2024-11-19T17:15:50,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/f3772feaa1c143b4b57cf21a72e419fd is 50, key is test_row_0/C:col10/1732036549939/Put/seqid=0 2024-11-19T17:15:50,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742345_1521 (size=12241) 2024-11-19T17:15:50,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742346_1522 (size=12151) 2024-11-19T17:15:50,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/f3772feaa1c143b4b57cf21a72e419fd 2024-11-19T17:15:50,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/5b004a0dad354150b6c9d9ae886f5a98 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/5b004a0dad354150b6c9d9ae886f5a98 2024-11-19T17:15:50,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:50,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036610734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:50,748 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:50,748 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:50,748 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:50,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/5b004a0dad354150b6c9d9ae886f5a98, entries=200, sequenceid=154, filesize=14.2 K 2024-11-19T17:15:50,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/d1114a6dcce44287899832b7e2e94931 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/d1114a6dcce44287899832b7e2e94931 2024-11-19T17:15:50,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/d1114a6dcce44287899832b7e2e94931, entries=150, sequenceid=154, filesize=11.9 K 2024-11-19T17:15:50,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/f3772feaa1c143b4b57cf21a72e419fd as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/f3772feaa1c143b4b57cf21a72e419fd 2024-11-19T17:15:50,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/f3772feaa1c143b4b57cf21a72e419fd, entries=150, sequenceid=154, filesize=11.9 K 2024-11-19T17:15:50,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 2d49416a7b2a16305b1ea9d2c71b4946 in 460ms, sequenceid=154, compaction requested=false 2024-11-19T17:15:50,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:50,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-19T17:15:50,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:50,841 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 2d49416a7b2a16305b1ea9d2c71b4946 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T17:15:50,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=A 2024-11-19T17:15:50,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:50,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=B 2024-11-19T17:15:50,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:50,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=C 2024-11-19T17:15:50,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:50,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/48ed4bc8ca114150b07c562ef3f3dc15 is 50, key is test_row_0/A:col10/1732036550329/Put/seqid=0 2024-11-19T17:15:50,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742347_1523 (size=12151) 2024-11-19T17:15:50,852 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/48ed4bc8ca114150b07c562ef3f3dc15 2024-11-19T17:15:50,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/bba8d888176f4dd9acd198bab662e207 is 50, key is test_row_0/B:col10/1732036550329/Put/seqid=0 2024-11-19T17:15:50,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742348_1524 (size=12151) 2024-11-19T17:15:50,878 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/bba8d888176f4dd9acd198bab662e207 2024-11-19T17:15:50,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/3d0fad72e2084308830ec4b787979173 is 50, key is test_row_0/C:col10/1732036550329/Put/seqid=0 2024-11-19T17:15:50,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742349_1525 (size=12151) 2024-11-19T17:15:50,904 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/3d0fad72e2084308830ec4b787979173 2024-11-19T17:15:50,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/48ed4bc8ca114150b07c562ef3f3dc15 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/48ed4bc8ca114150b07c562ef3f3dc15 2024-11-19T17:15:50,918 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/48ed4bc8ca114150b07c562ef3f3dc15, entries=150, sequenceid=166, filesize=11.9 K 2024-11-19T17:15:50,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/bba8d888176f4dd9acd198bab662e207 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/bba8d888176f4dd9acd198bab662e207 2024-11-19T17:15:50,925 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/bba8d888176f4dd9acd198bab662e207, entries=150, sequenceid=166, filesize=11.9 K 2024-11-19T17:15:50,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/3d0fad72e2084308830ec4b787979173 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/3d0fad72e2084308830ec4b787979173 2024-11-19T17:15:50,929 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/3d0fad72e2084308830ec4b787979173, entries=150, sequenceid=166, filesize=11.9 K 2024-11-19T17:15:50,930 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for 2d49416a7b2a16305b1ea9d2c71b4946 in 89ms, sequenceid=166, compaction requested=true 2024-11-19T17:15:50,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:50,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:50,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-19T17:15:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-19T17:15:50,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-19T17:15:50,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 857 msec 2024-11-19T17:15:50,940 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 864 msec 2024-11-19T17:15:51,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:15:51,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d49416a7b2a16305b1ea9d2c71b4946 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T17:15:51,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=A 2024-11-19T17:15:51,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:51,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=B 2024-11-19T17:15:51,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:51,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=C 2024-11-19T17:15:51,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:51,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/2a3abf5f4c7c41da8574c3487ce0fae4 is 50, key is test_row_0/A:col10/1732036551068/Put/seqid=0 2024-11-19T17:15:51,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742350_1526 (size=16931) 2024-11-19T17:15:51,146 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/adf4cd39e7b0475c9495d9afe2b76fa4 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/adf4cd39e7b0475c9495d9afe2b76fa4 2024-11-19T17:15:51,150 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2d49416a7b2a16305b1ea9d2c71b4946/C of 2d49416a7b2a16305b1ea9d2c71b4946 into adf4cd39e7b0475c9495d9afe2b76fa4(size=12.0 K), total size for store is 35.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:51,150 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:51,150 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., storeName=2d49416a7b2a16305b1ea9d2c71b4946/C, priority=12, startTime=1732036550288; duration=0sec 2024-11-19T17:15:51,150 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:51,150 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d49416a7b2a16305b1ea9d2c71b4946:C 2024-11-19T17:15:51,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036551142, max timestamp=1732036551165Num uniques 859; 2024-11-19T17:15:51,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036551142, max timestamp=1732036551165Num uniques 859; 2024-11-19T17:15:51,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:15:51,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:15:51,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036551142, max timestamp=1732036551165Num uniques 859; 2024-11-19T17:15:51,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:15:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T17:15:51,214 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:15:51,215 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:15:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-19T17:15:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T17:15:51,216 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:15:51,217 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:15:51,217 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:15:51,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036611226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:51,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:51,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:51,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:51,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036611228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:51,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:51,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:51,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:51,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:51,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036611237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:51,256 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:51,256 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:51,256 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T17:15:51,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036611343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:51,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:51,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:51,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036611353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:51,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:51,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:51,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:51,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:51,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T17:15:51,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:51,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:51,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:51,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:51,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:51,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036611364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:51,375 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:51,375 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:51,375 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:51,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/2a3abf5f4c7c41da8574c3487ce0fae4 2024-11-19T17:15:51,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/1e1e225701094a9980da093f7914c21c is 50, key is test_row_0/B:col10/1732036551068/Put/seqid=0 2024-11-19T17:15:51,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742351_1527 (size=12151) 2024-11-19T17:15:51,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T17:15:51,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:51,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:51,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:51,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:51,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T17:15:51,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036611563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:51,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036611563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:51,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:51,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:51,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:51,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036611584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:51,597 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:51,597 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:51,597 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:51,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T17:15:51,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:51,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:51,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:51,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:51,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:51,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T17:15:51,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:51,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:51,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:51,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:51,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:51,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T17:15:51,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036611883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:51,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:51,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:51,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036611883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:51,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:51,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:51,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:51,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:51,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/1e1e225701094a9980da093f7914c21c 2024-11-19T17:15:51,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/c8eea07bdc6d421fa3a014217ec76d3d is 50, key is test_row_0/C:col10/1732036551068/Put/seqid=0 2024-11-19T17:15:51,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:51,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036611903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:51,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742352_1528 (size=12151) 2024-11-19T17:15:51,919 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:51,919 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:51,919 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:51,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/c8eea07bdc6d421fa3a014217ec76d3d 2024-11-19T17:15:51,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/2a3abf5f4c7c41da8574c3487ce0fae4 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/2a3abf5f4c7c41da8574c3487ce0fae4 2024-11-19T17:15:51,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/2a3abf5f4c7c41da8574c3487ce0fae4, entries=250, sequenceid=176, filesize=16.5 K 2024-11-19T17:15:51,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/1e1e225701094a9980da093f7914c21c as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/1e1e225701094a9980da093f7914c21c 2024-11-19T17:15:51,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/1e1e225701094a9980da093f7914c21c, entries=150, sequenceid=176, filesize=11.9 K 2024-11-19T17:15:51,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/c8eea07bdc6d421fa3a014217ec76d3d as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/c8eea07bdc6d421fa3a014217ec76d3d 2024-11-19T17:15:51,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/c8eea07bdc6d421fa3a014217ec76d3d, entries=150, sequenceid=176, filesize=11.9 K 2024-11-19T17:15:51,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 2d49416a7b2a16305b1ea9d2c71b4946 in 846ms, sequenceid=176, compaction requested=true 2024-11-19T17:15:51,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:51,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d49416a7b2a16305b1ea9d2c71b4946:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:15:51,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:51,934 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:15:51,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d49416a7b2a16305b1ea9d2c71b4946:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:15:51,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:51,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d49416a7b2a16305b1ea9d2c71b4946:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:15:51,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:51,934 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:15:51,935 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55864 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:15:51,935 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:15:51,936 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 2d49416a7b2a16305b1ea9d2c71b4946/A is initiating minor compaction (all files) 2024-11-19T17:15:51,936 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 2d49416a7b2a16305b1ea9d2c71b4946/B is initiating minor compaction (all files) 2024-11-19T17:15:51,936 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d49416a7b2a16305b1ea9d2c71b4946/A in TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:51,936 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d49416a7b2a16305b1ea9d2c71b4946/B in TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:51,936 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/36257bb45aaf4632be48d9a543639a37, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/5b004a0dad354150b6c9d9ae886f5a98, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/48ed4bc8ca114150b07c562ef3f3dc15, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/2a3abf5f4c7c41da8574c3487ce0fae4] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp, totalSize=54.6 K 2024-11-19T17:15:51,936 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/3d00664c7dfa49ce9357c7910ce42fa4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/d1114a6dcce44287899832b7e2e94931, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/bba8d888176f4dd9acd198bab662e207, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/1e1e225701094a9980da093f7914c21c] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp, totalSize=47.6 K 2024-11-19T17:15:51,936 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36257bb45aaf4632be48d9a543639a37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732036549752 2024-11-19T17:15:51,936 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b004a0dad354150b6c9d9ae886f5a98, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732036549918 2024-11-19T17:15:51,936 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d00664c7dfa49ce9357c7910ce42fa4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732036549752 2024-11-19T17:15:51,937 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48ed4bc8ca114150b07c562ef3f3dc15, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732036550307 2024-11-19T17:15:51,937 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting d1114a6dcce44287899832b7e2e94931, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732036549921 2024-11-19T17:15:51,937 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a3abf5f4c7c41da8574c3487ce0fae4, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732036551055 2024-11-19T17:15:51,937 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting bba8d888176f4dd9acd198bab662e207, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732036550307 2024-11-19T17:15:51,937 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e1e225701094a9980da093f7914c21c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732036551068 2024-11-19T17:15:51,944 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d49416a7b2a16305b1ea9d2c71b4946#A#compaction#445 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:51,944 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d49416a7b2a16305b1ea9d2c71b4946#B#compaction#446 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:51,945 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/56a8e67f27994be798086497b67e4f8c is 50, key is test_row_0/A:col10/1732036551068/Put/seqid=0 2024-11-19T17:15:51,945 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/72a57aa8e0b542d3b76e696d5fc6601b is 50, key is test_row_0/B:col10/1732036551068/Put/seqid=0 2024-11-19T17:15:51,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742353_1529 (size=12527) 2024-11-19T17:15:51,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742354_1530 (size=12527) 2024-11-19T17:15:51,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T17:15:51,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:51,981 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 2d49416a7b2a16305b1ea9d2c71b4946 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-19T17:15:51,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=A 2024-11-19T17:15:51,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:51,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=B 2024-11-19T17:15:51,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:51,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=C 2024-11-19T17:15:51,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:51,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/a024492f5a0c44779900b09749873058 is 50, key is test_row_0/A:col10/1732036551216/Put/seqid=0 2024-11-19T17:15:51,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742355_1531 (size=12151) 2024-11-19T17:15:52,132 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T17:15:52,132 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-19T17:15:52,352 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/72a57aa8e0b542d3b76e696d5fc6601b as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/72a57aa8e0b542d3b76e696d5fc6601b 2024-11-19T17:15:52,352 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/56a8e67f27994be798086497b67e4f8c as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/56a8e67f27994be798086497b67e4f8c 2024-11-19T17:15:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T17:15:52,357 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2d49416a7b2a16305b1ea9d2c71b4946/A of 2d49416a7b2a16305b1ea9d2c71b4946 into 56a8e67f27994be798086497b67e4f8c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:52,357 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2d49416a7b2a16305b1ea9d2c71b4946/B of 2d49416a7b2a16305b1ea9d2c71b4946 into 72a57aa8e0b542d3b76e696d5fc6601b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:52,357 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:52,357 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:52,357 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., storeName=2d49416a7b2a16305b1ea9d2c71b4946/A, priority=12, startTime=1732036551934; duration=0sec 2024-11-19T17:15:52,357 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., storeName=2d49416a7b2a16305b1ea9d2c71b4946/B, priority=12, startTime=1732036551934; duration=0sec 2024-11-19T17:15:52,357 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:52,357 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d49416a7b2a16305b1ea9d2c71b4946:A 2024-11-19T17:15:52,357 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:52,357 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d49416a7b2a16305b1ea9d2c71b4946:B 2024-11-19T17:15:52,357 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:15:52,358 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:15:52,358 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 2d49416a7b2a16305b1ea9d2c71b4946/C is initiating minor compaction (all files) 2024-11-19T17:15:52,358 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d49416a7b2a16305b1ea9d2c71b4946/C in TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:52,358 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/adf4cd39e7b0475c9495d9afe2b76fa4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/f3772feaa1c143b4b57cf21a72e419fd, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/3d0fad72e2084308830ec4b787979173, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/c8eea07bdc6d421fa3a014217ec76d3d] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp, totalSize=47.6 K 2024-11-19T17:15:52,358 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting adf4cd39e7b0475c9495d9afe2b76fa4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732036549752 2024-11-19T17:15:52,359 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3772feaa1c143b4b57cf21a72e419fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732036549921 2024-11-19T17:15:52,359 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d0fad72e2084308830ec4b787979173, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732036550307 2024-11-19T17:15:52,359 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8eea07bdc6d421fa3a014217ec76d3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732036551068 2024-11-19T17:15:52,366 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d49416a7b2a16305b1ea9d2c71b4946#C#compaction#448 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:52,367 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/05738fe53bf449f194c6d5b80e4a2554 is 50, key is test_row_0/C:col10/1732036551068/Put/seqid=0 2024-11-19T17:15:52,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742356_1532 (size=12527) 2024-11-19T17:15:52,390 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/a024492f5a0c44779900b09749873058 2024-11-19T17:15:52,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/a1600446b9ea4a58b8f64869825e3fe7 is 50, key is test_row_0/B:col10/1732036551216/Put/seqid=0 2024-11-19T17:15:52,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742357_1533 (size=12151) 2024-11-19T17:15:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:15:52,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:52,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036612434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:52,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:52,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:52,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:52,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036612440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:52,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:52,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:52,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:52,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036612443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:52,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:52,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036612554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:52,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036612554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:52,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:52,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:52,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:52,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:52,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:52,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:52,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036612763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:52,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:52,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:52,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036612763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:52,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:52,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:52,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:52,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:52,778 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/05738fe53bf449f194c6d5b80e4a2554 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/05738fe53bf449f194c6d5b80e4a2554 2024-11-19T17:15:52,781 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2d49416a7b2a16305b1ea9d2c71b4946/C of 2d49416a7b2a16305b1ea9d2c71b4946 into 05738fe53bf449f194c6d5b80e4a2554(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:52,781 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:52,781 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., storeName=2d49416a7b2a16305b1ea9d2c71b4946/C, priority=12, startTime=1732036551934; duration=0sec 2024-11-19T17:15:52,782 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:52,782 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d49416a7b2a16305b1ea9d2c71b4946:C 2024-11-19T17:15:52,799 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/a1600446b9ea4a58b8f64869825e3fe7 2024-11-19T17:15:52,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/d930934bca5f442e83a90d87283bc23d is 50, key is test_row_0/C:col10/1732036551216/Put/seqid=0 2024-11-19T17:15:52,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742358_1534 (size=12151) 2024-11-19T17:15:52,810 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/d930934bca5f442e83a90d87283bc23d 2024-11-19T17:15:52,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/a024492f5a0c44779900b09749873058 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/a024492f5a0c44779900b09749873058 2024-11-19T17:15:52,817 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/a024492f5a0c44779900b09749873058, entries=150, sequenceid=213, filesize=11.9 K 2024-11-19T17:15:52,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/a1600446b9ea4a58b8f64869825e3fe7 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/a1600446b9ea4a58b8f64869825e3fe7 2024-11-19T17:15:52,821 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/a1600446b9ea4a58b8f64869825e3fe7, entries=150, sequenceid=213, filesize=11.9 K 2024-11-19T17:15:52,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/d930934bca5f442e83a90d87283bc23d as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/d930934bca5f442e83a90d87283bc23d 2024-11-19T17:15:52,825 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/d930934bca5f442e83a90d87283bc23d, entries=150, sequenceid=213, filesize=11.9 K 2024-11-19T17:15:52,825 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 2d49416a7b2a16305b1ea9d2c71b4946 in 844ms, sequenceid=213, compaction requested=false 2024-11-19T17:15:52,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:52,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:52,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-19T17:15:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-19T17:15:52,829 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-19T17:15:52,829 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6100 sec 2024-11-19T17:15:52,831 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.6150 sec 2024-11-19T17:15:53,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:15:53,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d49416a7b2a16305b1ea9d2c71b4946 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T17:15:53,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=A 2024-11-19T17:15:53,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:53,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=B 2024-11-19T17:15:53,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:53,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=C 2024-11-19T17:15:53,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:53,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/1b91af9dd89d4e099d55cec1357ae453 is 50, key is test_row_0/A:col10/1732036552431/Put/seqid=0 2024-11-19T17:15:53,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742359_1535 (size=14541) 2024-11-19T17:15:53,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:53,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036613164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:53,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:53,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:53,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:53,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:53,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036613169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:53,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:53,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:53,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:53,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:53,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036613273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:53,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:53,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:53,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:53,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:53,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036613284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:53,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:53,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:53,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:53,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T17:15:53,374 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:15:53,374 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:15:53,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-19T17:15:53,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T17:15:53,376 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:15:53,377 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:15:53,377 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:15:53,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:53,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036613453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:53,464 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_2' in 2d49416a7b2a16305b1ea9d2c71b4946 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:15:53,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:53,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:53,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:53,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T17:15:53,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/1b91af9dd89d4e099d55cec1357ae453 2024-11-19T17:15:53,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:53,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036613494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:53,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:53,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:53,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:53,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/de8a64b17e244f46a8c9e4c113e1ac06 is 50, key is test_row_0/B:col10/1732036552431/Put/seqid=0 2024-11-19T17:15:53,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742360_1536 (size=12151) 2024-11-19T17:15:53,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:53,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036613504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:53,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-19T17:15:53,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:53,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:53,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:53,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:53,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:53,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:53,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-19T17:15:53,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:53,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:53,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:53,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:53,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:53,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:53,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T17:15:53,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:53,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036613814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:53,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:53,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:53,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:53,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:53,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036613824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:53,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:53,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:53,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:53,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-19T17:15:53,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:53,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:53,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:53,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:53,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:53,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:53,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/de8a64b17e244f46a8c9e4c113e1ac06 2024-11-19T17:15:53,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/a0b7dcac19b643adaa80e00faf629b31 is 50, key is test_row_0/C:col10/1732036552431/Put/seqid=0 2024-11-19T17:15:53,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742361_1537 (size=12151) 2024-11-19T17:15:53,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/a0b7dcac19b643adaa80e00faf629b31 2024-11-19T17:15:53,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/1b91af9dd89d4e099d55cec1357ae453 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/1b91af9dd89d4e099d55cec1357ae453 2024-11-19T17:15:53,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/1b91af9dd89d4e099d55cec1357ae453, entries=200, sequenceid=228, filesize=14.2 K 2024-11-19T17:15:53,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/de8a64b17e244f46a8c9e4c113e1ac06 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/de8a64b17e244f46a8c9e4c113e1ac06 2024-11-19T17:15:53,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/de8a64b17e244f46a8c9e4c113e1ac06, entries=150, sequenceid=228, filesize=11.9 K 2024-11-19T17:15:53,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/a0b7dcac19b643adaa80e00faf629b31 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/a0b7dcac19b643adaa80e00faf629b31 2024-11-19T17:15:53,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/a0b7dcac19b643adaa80e00faf629b31, entries=150, sequenceid=228, filesize=11.9 K 2024-11-19T17:15:53,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 2d49416a7b2a16305b1ea9d2c71b4946 in 845ms, sequenceid=228, compaction requested=true 2024-11-19T17:15:53,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:53,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d49416a7b2a16305b1ea9d2c71b4946:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:15:53,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:53,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d49416a7b2a16305b1ea9d2c71b4946:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:15:53,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:53,934 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:53,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d49416a7b2a16305b1ea9d2c71b4946:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:15:53,934 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:53,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:53,935 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:53,935 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:53,935 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 2d49416a7b2a16305b1ea9d2c71b4946/B is initiating minor compaction (all files) 2024-11-19T17:15:53,935 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 2d49416a7b2a16305b1ea9d2c71b4946/A is initiating minor compaction (all files) 2024-11-19T17:15:53,935 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d49416a7b2a16305b1ea9d2c71b4946/B in TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:53,935 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d49416a7b2a16305b1ea9d2c71b4946/A in TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:53,935 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/72a57aa8e0b542d3b76e696d5fc6601b, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/a1600446b9ea4a58b8f64869825e3fe7, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/de8a64b17e244f46a8c9e4c113e1ac06] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp, totalSize=36.0 K 2024-11-19T17:15:53,935 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/56a8e67f27994be798086497b67e4f8c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/a024492f5a0c44779900b09749873058, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/1b91af9dd89d4e099d55cec1357ae453] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp, totalSize=38.3 K 2024-11-19T17:15:53,935 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 72a57aa8e0b542d3b76e696d5fc6601b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732036551068 2024-11-19T17:15:53,935 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56a8e67f27994be798086497b67e4f8c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732036551068 2024-11-19T17:15:53,935 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting a024492f5a0c44779900b09749873058, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732036551216 2024-11-19T17:15:53,935 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting a1600446b9ea4a58b8f64869825e3fe7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732036551216 2024-11-19T17:15:53,936 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting de8a64b17e244f46a8c9e4c113e1ac06, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732036552431 2024-11-19T17:15:53,936 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b91af9dd89d4e099d55cec1357ae453, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732036552431 2024-11-19T17:15:53,942 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d49416a7b2a16305b1ea9d2c71b4946#B#compaction#455 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:53,942 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d49416a7b2a16305b1ea9d2c71b4946#A#compaction#454 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:53,943 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/16545560aea04d73a6c1a8b542310063 is 50, key is test_row_0/B:col10/1732036552431/Put/seqid=0 2024-11-19T17:15:53,943 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/eaf1b21566a949d29f9517a658328387 is 50, key is test_row_0/A:col10/1732036552431/Put/seqid=0 2024-11-19T17:15:53,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742362_1538 (size=12629) 2024-11-19T17:15:53,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742363_1539 (size=12629) 2024-11-19T17:15:53,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-19T17:15:53,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:53,988 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 2d49416a7b2a16305b1ea9d2c71b4946 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T17:15:53,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=A 2024-11-19T17:15:53,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:53,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=B 2024-11-19T17:15:53,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:53,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=C 2024-11-19T17:15:53,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:53,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/8421315e1d2e4bb591d05f4f350825aa is 50, key is test_row_0/A:col10/1732036553162/Put/seqid=0 2024-11-19T17:15:53,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742364_1540 (size=12151) 2024-11-19T17:15:54,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T17:15:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:15:54,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:54,352 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/16545560aea04d73a6c1a8b542310063 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/16545560aea04d73a6c1a8b542310063 2024-11-19T17:15:54,352 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/eaf1b21566a949d29f9517a658328387 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/eaf1b21566a949d29f9517a658328387 2024-11-19T17:15:54,356 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d49416a7b2a16305b1ea9d2c71b4946/A of 2d49416a7b2a16305b1ea9d2c71b4946 into eaf1b21566a949d29f9517a658328387(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:54,356 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d49416a7b2a16305b1ea9d2c71b4946/B of 2d49416a7b2a16305b1ea9d2c71b4946 into 16545560aea04d73a6c1a8b542310063(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:54,356 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:54,356 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:54,356 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., storeName=2d49416a7b2a16305b1ea9d2c71b4946/A, priority=13, startTime=1732036553934; duration=0sec 2024-11-19T17:15:54,356 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., storeName=2d49416a7b2a16305b1ea9d2c71b4946/B, priority=13, startTime=1732036553934; duration=0sec 2024-11-19T17:15:54,357 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:54,357 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d49416a7b2a16305b1ea9d2c71b4946:A 2024-11-19T17:15:54,357 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:54,357 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d49416a7b2a16305b1ea9d2c71b4946:B 2024-11-19T17:15:54,357 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:54,357 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:54,357 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 2d49416a7b2a16305b1ea9d2c71b4946/C is initiating minor compaction (all files) 2024-11-19T17:15:54,357 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d49416a7b2a16305b1ea9d2c71b4946/C in TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:54,357 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/05738fe53bf449f194c6d5b80e4a2554, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/d930934bca5f442e83a90d87283bc23d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/a0b7dcac19b643adaa80e00faf629b31] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp, totalSize=36.0 K 2024-11-19T17:15:54,358 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05738fe53bf449f194c6d5b80e4a2554, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732036551068 2024-11-19T17:15:54,358 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting d930934bca5f442e83a90d87283bc23d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732036551216 2024-11-19T17:15:54,358 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0b7dcac19b643adaa80e00faf629b31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732036552431 2024-11-19T17:15:54,363 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d49416a7b2a16305b1ea9d2c71b4946#C#compaction#457 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:54,364 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/91f7582d059240a49d7c15fa9a54c52d is 50, key is test_row_0/C:col10/1732036552431/Put/seqid=0 2024-11-19T17:15:54,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742365_1541 (size=12629) 2024-11-19T17:15:54,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036614365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:54,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:54,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:54,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:54,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036614376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:54,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:54,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:54,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:54,396 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/8421315e1d2e4bb591d05f4f350825aa 2024-11-19T17:15:54,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/1de74ab3e9734d80815eced24a576aaf is 50, key is test_row_0/B:col10/1732036553162/Put/seqid=0 2024-11-19T17:15:54,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742366_1542 (size=12151) 2024-11-19T17:15:54,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036614483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:54,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036614493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:54,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:54,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:54,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T17:15:54,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51512 deadline: 1732036614664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:54,671 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_1' in 2d49416a7b2a16305b1ea9d2c71b4946 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8265 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:15:54,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:54,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:54,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:54,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51578 deadline: 1732036614683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:54,692 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_2' in 2d49416a7b2a16305b1ea9d2c71b4946 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8273 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:15:54,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:54,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:54,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:54,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036614693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:54,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:54,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:54,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036614704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:54,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:54,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:54,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:54,771 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/91f7582d059240a49d7c15fa9a54c52d as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/91f7582d059240a49d7c15fa9a54c52d 2024-11-19T17:15:54,774 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d49416a7b2a16305b1ea9d2c71b4946/C of 2d49416a7b2a16305b1ea9d2c71b4946 into 91f7582d059240a49d7c15fa9a54c52d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:54,774 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:54,774 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., storeName=2d49416a7b2a16305b1ea9d2c71b4946/C, priority=13, startTime=1732036553934; duration=0sec 2024-11-19T17:15:54,775 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:54,775 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d49416a7b2a16305b1ea9d2c71b4946:C 2024-11-19T17:15:54,808 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/1de74ab3e9734d80815eced24a576aaf 2024-11-19T17:15:54,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/95b9df38a0024d3a8de3c940d0d8481a is 50, key is test_row_0/C:col10/1732036553162/Put/seqid=0 2024-11-19T17:15:54,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742367_1543 (size=12151) 2024-11-19T17:15:55,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036615014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:55,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:55,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:55,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036615014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:55,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:55,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:55,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:55,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:55,218 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/95b9df38a0024d3a8de3c940d0d8481a 2024-11-19T17:15:55,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/8421315e1d2e4bb591d05f4f350825aa as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/8421315e1d2e4bb591d05f4f350825aa 2024-11-19T17:15:55,231 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/8421315e1d2e4bb591d05f4f350825aa, entries=150, sequenceid=248, filesize=11.9 K 2024-11-19T17:15:55,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/1de74ab3e9734d80815eced24a576aaf as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/1de74ab3e9734d80815eced24a576aaf 2024-11-19T17:15:55,235 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/1de74ab3e9734d80815eced24a576aaf, entries=150, sequenceid=248, filesize=11.9 K 2024-11-19T17:15:55,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/95b9df38a0024d3a8de3c940d0d8481a as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/95b9df38a0024d3a8de3c940d0d8481a 2024-11-19T17:15:55,238 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/95b9df38a0024d3a8de3c940d0d8481a, entries=150, sequenceid=248, filesize=11.9 K 2024-11-19T17:15:55,239 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 2d49416a7b2a16305b1ea9d2c71b4946 in 1251ms, sequenceid=248, compaction requested=false 2024-11-19T17:15:55,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:55,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:55,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-19T17:15:55,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-19T17:15:55,242 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-19T17:15:55,242 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8630 sec 2024-11-19T17:15:55,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.8690 sec 2024-11-19T17:15:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:15:55,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d49416a7b2a16305b1ea9d2c71b4946 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T17:15:55,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=A 2024-11-19T17:15:55,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:55,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=B 2024-11-19T17:15:55,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:55,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=C 2024-11-19T17:15:55,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:55,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/57a3da48c0044894a172ddc4a94855ac is 50, key is test_row_0/A:col10/1732036554353/Put/seqid=0 2024-11-19T17:15:55,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742368_1544 (size=14691) 2024-11-19T17:15:55,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T17:15:55,524 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:15:55,524 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:15:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-19T17:15:55,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T17:15:55,526 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:15:55,527 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:15:55,527 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:15:55,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036615571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:55,580 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:55,580 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:55,580 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:55,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036615573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:55,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036615579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T17:15:55,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T17:15:55,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:55,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:55,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:55,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:55,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:55,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036615683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:55,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:55,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:55,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:55,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036615694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:55,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:55,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:55,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:55,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036615704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:55,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:55,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:55,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:55,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T17:15:55,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:55,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:55,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:55,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:55,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T17:15:55,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/57a3da48c0044894a172ddc4a94855ac 2024-11-19T17:15:55,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/273eaf40f68044b6bfb346dd652711e2 is 50, key is test_row_0/B:col10/1732036554353/Put/seqid=0 2024-11-19T17:15:55,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036615903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:55,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:55,916 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:55,916 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036615903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:55,916 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:55,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:55,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:55,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:55,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742369_1545 (size=12251) 2024-11-19T17:15:55,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036615924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:55,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:55,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:55,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:55,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T17:15:55,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:55,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:55,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:55,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:55,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:56,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T17:15:56,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:56,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:56,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:56,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:56,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:56,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:56,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T17:15:56,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:56,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036616223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:56,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:56,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:56,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:56,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036616223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:56,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:56,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:56,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:56,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:56,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:56,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036616244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:56,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T17:15:56,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:56,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:56,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:56,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:56,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:56,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/273eaf40f68044b6bfb346dd652711e2 2024-11-19T17:15:56,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/c2adcb7775ee49e2a4403947dbec6948 is 50, key is test_row_0/C:col10/1732036554353/Put/seqid=0 2024-11-19T17:15:56,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742370_1546 (size=12251) 2024-11-19T17:15:56,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T17:15:56,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:56,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:56,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:56,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:56,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:56,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T17:15:56,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:56,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:56,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:56,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:56,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:56,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T17:15:56,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/c2adcb7775ee49e2a4403947dbec6948 2024-11-19T17:15:56,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/57a3da48c0044894a172ddc4a94855ac as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/57a3da48c0044894a172ddc4a94855ac 2024-11-19T17:15:56,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/57a3da48c0044894a172ddc4a94855ac, entries=200, sequenceid=264, filesize=14.3 K 2024-11-19T17:15:56,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/273eaf40f68044b6bfb346dd652711e2 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/273eaf40f68044b6bfb346dd652711e2 2024-11-19T17:15:56,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/273eaf40f68044b6bfb346dd652711e2, entries=150, sequenceid=264, filesize=12.0 K 2024-11-19T17:15:56,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/c2adcb7775ee49e2a4403947dbec6948 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/c2adcb7775ee49e2a4403947dbec6948 2024-11-19T17:15:56,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/c2adcb7775ee49e2a4403947dbec6948, entries=150, sequenceid=264, filesize=12.0 K 2024-11-19T17:15:56,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T17:15:56,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:56,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:56,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 2d49416a7b2a16305b1ea9d2c71b4946 in 1250ms, sequenceid=264, compaction requested=true 2024-11-19T17:15:56,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:56,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:56,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:56,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d49416a7b2a16305b1ea9d2c71b4946:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:15:56,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:56,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:56,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d49416a7b2a16305b1ea9d2c71b4946:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:15:56,751 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:56,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:56,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d49416a7b2a16305b1ea9d2c71b4946:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:15:56,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:56,751 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:56,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:15:56,752 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:56,752 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37031 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:56,752 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 2d49416a7b2a16305b1ea9d2c71b4946/A is initiating minor compaction (all files) 2024-11-19T17:15:56,752 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 2d49416a7b2a16305b1ea9d2c71b4946/B is initiating minor compaction (all files) 2024-11-19T17:15:56,752 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d49416a7b2a16305b1ea9d2c71b4946/A in TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:56,752 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d49416a7b2a16305b1ea9d2c71b4946/B in TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:56,752 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/eaf1b21566a949d29f9517a658328387, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/8421315e1d2e4bb591d05f4f350825aa, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/57a3da48c0044894a172ddc4a94855ac] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp, totalSize=38.5 K 2024-11-19T17:15:56,752 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/16545560aea04d73a6c1a8b542310063, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/1de74ab3e9734d80815eced24a576aaf, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/273eaf40f68044b6bfb346dd652711e2] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp, totalSize=36.2 K 2024-11-19T17:15:56,753 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting eaf1b21566a949d29f9517a658328387, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732036552431 2024-11-19T17:15:56,754 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8421315e1d2e4bb591d05f4f350825aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732036553138 2024-11-19T17:15:56,754 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 16545560aea04d73a6c1a8b542310063, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732036552431 2024-11-19T17:15:56,754 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 1de74ab3e9734d80815eced24a576aaf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732036553138 2024-11-19T17:15:56,754 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57a3da48c0044894a172ddc4a94855ac, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732036554353 2024-11-19T17:15:56,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:15:56,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d49416a7b2a16305b1ea9d2c71b4946 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T17:15:56,755 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 273eaf40f68044b6bfb346dd652711e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732036554353 2024-11-19T17:15:56,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=A 2024-11-19T17:15:56,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:56,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=B 2024-11-19T17:15:56,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:56,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=C 2024-11-19T17:15:56,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:56,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/7ab35b3c046a43c5b30217f87b262cdd is 50, key is test_row_0/A:col10/1732036555560/Put/seqid=0 2024-11-19T17:15:56,762 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d49416a7b2a16305b1ea9d2c71b4946#A#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:56,763 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d49416a7b2a16305b1ea9d2c71b4946#B#compaction#464 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:56,763 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/e501b554672c4277919a20b6640d0513 is 50, key is test_row_0/A:col10/1732036554353/Put/seqid=0 2024-11-19T17:15:56,763 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/50a27562768b4a389325a10fd96c5d85 is 50, key is test_row_0/B:col10/1732036554353/Put/seqid=0 2024-11-19T17:15:56,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742371_1547 (size=14741) 2024-11-19T17:15:56,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/7ab35b3c046a43c5b30217f87b262cdd 2024-11-19T17:15:56,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/5550ce08efaa4caa8e21f655dcd2b6c4 is 50, key is test_row_0/B:col10/1732036555560/Put/seqid=0 2024-11-19T17:15:56,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742373_1549 (size=12831) 2024-11-19T17:15:56,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742372_1548 (size=12831) 2024-11-19T17:15:56,783 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/e501b554672c4277919a20b6640d0513 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/e501b554672c4277919a20b6640d0513 2024-11-19T17:15:56,787 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/50a27562768b4a389325a10fd96c5d85 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/50a27562768b4a389325a10fd96c5d85 2024-11-19T17:15:56,790 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d49416a7b2a16305b1ea9d2c71b4946/A of 2d49416a7b2a16305b1ea9d2c71b4946 into e501b554672c4277919a20b6640d0513(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:56,790 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:56,790 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., storeName=2d49416a7b2a16305b1ea9d2c71b4946/A, priority=13, startTime=1732036556751; duration=0sec 2024-11-19T17:15:56,790 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:56,790 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d49416a7b2a16305b1ea9d2c71b4946:A 2024-11-19T17:15:56,790 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:15:56,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742374_1550 (size=12301) 2024-11-19T17:15:56,793 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37031 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:15:56,793 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d49416a7b2a16305b1ea9d2c71b4946/B of 2d49416a7b2a16305b1ea9d2c71b4946 into 50a27562768b4a389325a10fd96c5d85(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:56,793 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 2d49416a7b2a16305b1ea9d2c71b4946/C is initiating minor compaction (all files) 2024-11-19T17:15:56,793 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:56,793 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., storeName=2d49416a7b2a16305b1ea9d2c71b4946/B, priority=13, startTime=1732036556751; duration=0sec 2024-11-19T17:15:56,793 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d49416a7b2a16305b1ea9d2c71b4946/C in TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:56,793 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:56,794 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d49416a7b2a16305b1ea9d2c71b4946:B 2024-11-19T17:15:56,794 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/91f7582d059240a49d7c15fa9a54c52d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/95b9df38a0024d3a8de3c940d0d8481a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/c2adcb7775ee49e2a4403947dbec6948] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp, totalSize=36.2 K 2024-11-19T17:15:56,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/5550ce08efaa4caa8e21f655dcd2b6c4 2024-11-19T17:15:56,794 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91f7582d059240a49d7c15fa9a54c52d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732036552431 2024-11-19T17:15:56,794 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95b9df38a0024d3a8de3c940d0d8481a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732036553138 2024-11-19T17:15:56,795 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2adcb7775ee49e2a4403947dbec6948, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732036554353 2024-11-19T17:15:56,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:56,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036616786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:56,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:56,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:56,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:56,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/e6d3531317954c37a74cea6e91ea0de9 is 50, key is test_row_0/C:col10/1732036555560/Put/seqid=0 2024-11-19T17:15:56,804 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d49416a7b2a16305b1ea9d2c71b4946#C#compaction#468 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:56,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:56,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036616788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:56,804 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/83eab07641494e8e8352a86f8d0d2f98 is 50, key is test_row_0/C:col10/1732036554353/Put/seqid=0 2024-11-19T17:15:56,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:56,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036616789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:56,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:56,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:56,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:56,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742375_1551 (size=12301) 2024-11-19T17:15:56,806 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:56,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:56,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:56,807 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/e6d3531317954c37a74cea6e91ea0de9 2024-11-19T17:15:56,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742376_1552 (size=12831) 2024-11-19T17:15:56,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/7ab35b3c046a43c5b30217f87b262cdd as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/7ab35b3c046a43c5b30217f87b262cdd 2024-11-19T17:15:56,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/7ab35b3c046a43c5b30217f87b262cdd, entries=200, sequenceid=284, filesize=14.4 K 2024-11-19T17:15:56,814 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/83eab07641494e8e8352a86f8d0d2f98 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/83eab07641494e8e8352a86f8d0d2f98 2024-11-19T17:15:56,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/5550ce08efaa4caa8e21f655dcd2b6c4 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/5550ce08efaa4caa8e21f655dcd2b6c4 2024-11-19T17:15:56,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/5550ce08efaa4caa8e21f655dcd2b6c4, entries=150, sequenceid=284, filesize=12.0 K 2024-11-19T17:15:56,819 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d49416a7b2a16305b1ea9d2c71b4946/C of 2d49416a7b2a16305b1ea9d2c71b4946 into 83eab07641494e8e8352a86f8d0d2f98(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:56,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/e6d3531317954c37a74cea6e91ea0de9 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/e6d3531317954c37a74cea6e91ea0de9 2024-11-19T17:15:56,819 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:56,820 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., storeName=2d49416a7b2a16305b1ea9d2c71b4946/C, priority=13, startTime=1732036556751; duration=0sec 2024-11-19T17:15:56,820 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:56,820 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d49416a7b2a16305b1ea9d2c71b4946:C 2024-11-19T17:15:56,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/e6d3531317954c37a74cea6e91ea0de9, entries=150, sequenceid=284, filesize=12.0 K 2024-11-19T17:15:56,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 2d49416a7b2a16305b1ea9d2c71b4946 in 69ms, sequenceid=284, compaction requested=false 2024-11-19T17:15:56,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:56,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T17:15:56,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:56,904 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 2d49416a7b2a16305b1ea9d2c71b4946 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T17:15:56,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=A 2024-11-19T17:15:56,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:56,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=B 2024-11-19T17:15:56,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:56,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=C 2024-11-19T17:15:56,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:56,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/33b784943c114525aec7f6713d995f13 is 50, key is test_row_0/A:col10/1732036556788/Put/seqid=0 2024-11-19T17:15:56,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:15:56,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:15:56,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742377_1553 (size=12301) 2024-11-19T17:15:56,913 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/33b784943c114525aec7f6713d995f13 2024-11-19T17:15:56,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/68c26cc6f4b64ace9e423a599a230d3e is 50, key is test_row_0/B:col10/1732036556788/Put/seqid=0 2024-11-19T17:15:56,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742378_1554 (size=12301) 2024-11-19T17:15:56,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:56,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036616969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:56,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:56,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:56,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:56,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036616976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:56,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036616976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:56,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:56,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:56,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:56,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:56,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:56,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:57,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036617083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:57,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:57,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:57,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:57,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036617094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:57,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:57,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:57,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:57,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036617094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:57,102 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:57,102 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:57,102 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:57,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036617303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:57,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:57,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:57,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:57,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036617314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:57,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036617314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:57,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:57,331 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/68c26cc6f4b64ace9e423a599a230d3e 2024-11-19T17:15:57,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/7e21052ce3c64cd985c2765326bfaff4 is 50, key is test_row_0/C:col10/1732036556788/Put/seqid=0 2024-11-19T17:15:57,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742379_1555 (size=12301) 2024-11-19T17:15:57,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:57,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036617623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:57,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:57,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:57,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:57,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:57,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036617633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:57,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:57,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:57,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036617634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:57,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:57,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:57,645 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:57,645 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:57,645 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:57,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T17:15:57,748 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/7e21052ce3c64cd985c2765326bfaff4 2024-11-19T17:15:57,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/33b784943c114525aec7f6713d995f13 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/33b784943c114525aec7f6713d995f13 2024-11-19T17:15:57,756 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/33b784943c114525aec7f6713d995f13, entries=150, sequenceid=299, filesize=12.0 K 2024-11-19T17:15:57,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/68c26cc6f4b64ace9e423a599a230d3e as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/68c26cc6f4b64ace9e423a599a230d3e 2024-11-19T17:15:57,761 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/68c26cc6f4b64ace9e423a599a230d3e, entries=150, sequenceid=299, filesize=12.0 K 2024-11-19T17:15:57,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/7e21052ce3c64cd985c2765326bfaff4 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/7e21052ce3c64cd985c2765326bfaff4 2024-11-19T17:15:57,764 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/7e21052ce3c64cd985c2765326bfaff4, entries=150, sequenceid=299, filesize=12.0 K 2024-11-19T17:15:57,765 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 2d49416a7b2a16305b1ea9d2c71b4946 in 861ms, sequenceid=299, compaction requested=true 2024-11-19T17:15:57,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:57,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:57,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-19T17:15:57,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-19T17:15:57,768 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-19T17:15:57,768 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2390 sec 2024-11-19T17:15:57,770 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 2.2450 sec 2024-11-19T17:15:58,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:15:58,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d49416a7b2a16305b1ea9d2c71b4946 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-19T17:15:58,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=A 2024-11-19T17:15:58,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:58,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=B 2024-11-19T17:15:58,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:58,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=C 2024-11-19T17:15:58,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:15:58,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/d03adaa7770b4c6a995c42f5cb0f53ce is 50, key is test_row_0/A:col10/1732036556975/Put/seqid=0 2024-11-19T17:15:58,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742380_1556 (size=14741) 2024-11-19T17:15:58,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/d03adaa7770b4c6a995c42f5cb0f53ce 2024-11-19T17:15:58,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/d3468040c2ea428e8e77f36845664096 is 50, key is test_row_0/B:col10/1732036556975/Put/seqid=0 2024-11-19T17:15:58,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742381_1557 (size=12301) 2024-11-19T17:15:58,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:58,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:58,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036618181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:58,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036618180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:58,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:58,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036618189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:58,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:58,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036618304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:58,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:58,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036618304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:58,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:58,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036618304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:58,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:58,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036618524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:58,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:58,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036618524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:58,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:58,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:58,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:58,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:58,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:58,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036618524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:58,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:58,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:58,528 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:58,528 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:58,528 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:58,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/d3468040c2ea428e8e77f36845664096 2024-11-19T17:15:58,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/cb36d69ab8eb46c8849cf0540710388c is 50, key is test_row_0/C:col10/1732036556975/Put/seqid=0 2024-11-19T17:15:58,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742382_1558 (size=12301) 2024-11-19T17:15:58,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/cb36d69ab8eb46c8849cf0540710388c 2024-11-19T17:15:58,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/d03adaa7770b4c6a995c42f5cb0f53ce as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/d03adaa7770b4c6a995c42f5cb0f53ce 2024-11-19T17:15:58,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/d03adaa7770b4c6a995c42f5cb0f53ce, entries=200, sequenceid=320, filesize=14.4 K 2024-11-19T17:15:58,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/d3468040c2ea428e8e77f36845664096 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/d3468040c2ea428e8e77f36845664096 2024-11-19T17:15:58,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/d3468040c2ea428e8e77f36845664096, entries=150, sequenceid=320, filesize=12.0 K 2024-11-19T17:15:58,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/cb36d69ab8eb46c8849cf0540710388c as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/cb36d69ab8eb46c8849cf0540710388c 2024-11-19T17:15:58,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/cb36d69ab8eb46c8849cf0540710388c, entries=150, sequenceid=320, filesize=12.0 K 2024-11-19T17:15:58,601 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 2d49416a7b2a16305b1ea9d2c71b4946 in 452ms, sequenceid=320, compaction requested=true 2024-11-19T17:15:58,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:58,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d49416a7b2a16305b1ea9d2c71b4946:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:15:58,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:58,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d49416a7b2a16305b1ea9d2c71b4946:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:15:58,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:58,601 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:15:58,601 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:15:58,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d49416a7b2a16305b1ea9d2c71b4946:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:15:58,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:58,602 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49734 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:15:58,602 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54614 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:15:58,602 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 2d49416a7b2a16305b1ea9d2c71b4946/A is initiating minor compaction (all files) 2024-11-19T17:15:58,602 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 2d49416a7b2a16305b1ea9d2c71b4946/B is initiating minor compaction (all files) 2024-11-19T17:15:58,602 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d49416a7b2a16305b1ea9d2c71b4946/A in TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:58,602 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d49416a7b2a16305b1ea9d2c71b4946/B in TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:58,602 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/e501b554672c4277919a20b6640d0513, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/7ab35b3c046a43c5b30217f87b262cdd, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/33b784943c114525aec7f6713d995f13, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/d03adaa7770b4c6a995c42f5cb0f53ce] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp, totalSize=53.3 K 2024-11-19T17:15:58,602 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/50a27562768b4a389325a10fd96c5d85, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/5550ce08efaa4caa8e21f655dcd2b6c4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/68c26cc6f4b64ace9e423a599a230d3e, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/d3468040c2ea428e8e77f36845664096] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp, totalSize=48.6 K 2024-11-19T17:15:58,603 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 50a27562768b4a389325a10fd96c5d85, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732036554353 2024-11-19T17:15:58,603 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting e501b554672c4277919a20b6640d0513, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732036554353 2024-11-19T17:15:58,603 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 5550ce08efaa4caa8e21f655dcd2b6c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732036555560 2024-11-19T17:15:58,603 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ab35b3c046a43c5b30217f87b262cdd, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732036555560 2024-11-19T17:15:58,603 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33b784943c114525aec7f6713d995f13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732036556782 2024-11-19T17:15:58,603 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 68c26cc6f4b64ace9e423a599a230d3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732036556782 2024-11-19T17:15:58,603 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting d03adaa7770b4c6a995c42f5cb0f53ce, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732036556950 2024-11-19T17:15:58,603 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting d3468040c2ea428e8e77f36845664096, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732036556950 2024-11-19T17:15:58,612 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d49416a7b2a16305b1ea9d2c71b4946#B#compaction#475 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:58,613 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/59c6a4bac5e045558a98308a772d4582 is 50, key is test_row_0/B:col10/1732036556975/Put/seqid=0 2024-11-19T17:15:58,615 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d49416a7b2a16305b1ea9d2c71b4946#A#compaction#476 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:58,615 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/1ae26d7fb4174c85b75807da00c6ee68 is 50, key is test_row_0/A:col10/1732036556975/Put/seqid=0 2024-11-19T17:15:58,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742383_1559 (size=13017) 2024-11-19T17:15:58,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742384_1560 (size=13017) 2024-11-19T17:15:58,631 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/1ae26d7fb4174c85b75807da00c6ee68 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/1ae26d7fb4174c85b75807da00c6ee68 2024-11-19T17:15:58,635 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2d49416a7b2a16305b1ea9d2c71b4946/A of 2d49416a7b2a16305b1ea9d2c71b4946 into 1ae26d7fb4174c85b75807da00c6ee68(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:58,635 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:58,635 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., storeName=2d49416a7b2a16305b1ea9d2c71b4946/A, priority=12, startTime=1732036558601; duration=0sec 2024-11-19T17:15:58,636 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:15:58,636 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d49416a7b2a16305b1ea9d2c71b4946:A 2024-11-19T17:15:58,636 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:15:58,637 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49734 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:15:58,637 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 2d49416a7b2a16305b1ea9d2c71b4946/C is initiating minor compaction (all files) 2024-11-19T17:15:58,637 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d49416a7b2a16305b1ea9d2c71b4946/C in TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:58,638 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/83eab07641494e8e8352a86f8d0d2f98, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/e6d3531317954c37a74cea6e91ea0de9, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/7e21052ce3c64cd985c2765326bfaff4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/cb36d69ab8eb46c8849cf0540710388c] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp, totalSize=48.6 K 2024-11-19T17:15:58,638 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83eab07641494e8e8352a86f8d0d2f98, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732036554353 2024-11-19T17:15:58,638 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6d3531317954c37a74cea6e91ea0de9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732036555560 2024-11-19T17:15:58,638 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e21052ce3c64cd985c2765326bfaff4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732036556782 2024-11-19T17:15:58,639 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb36d69ab8eb46c8849cf0540710388c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732036556950 2024-11-19T17:15:58,648 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d49416a7b2a16305b1ea9d2c71b4946#C#compaction#477 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:15:58,649 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/af254ce784584828b33832d5c44cd801 is 50, key is test_row_0/C:col10/1732036556975/Put/seqid=0 2024-11-19T17:15:58,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742385_1561 (size=13017) 2024-11-19T17:15:58,664 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/af254ce784584828b33832d5c44cd801 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/af254ce784584828b33832d5c44cd801 2024-11-19T17:15:58,672 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2d49416a7b2a16305b1ea9d2c71b4946/C of 2d49416a7b2a16305b1ea9d2c71b4946 into af254ce784584828b33832d5c44cd801(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:58,672 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:58,672 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., storeName=2d49416a7b2a16305b1ea9d2c71b4946/C, priority=12, startTime=1732036558601; duration=0sec 2024-11-19T17:15:58,672 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:58,672 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d49416a7b2a16305b1ea9d2c71b4946:C 2024-11-19T17:15:58,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:15:58,837 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d49416a7b2a16305b1ea9d2c71b4946 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T17:15:58,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=A 2024-11-19T17:15:58,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:58,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=B 2024-11-19T17:15:58,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:58,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=C 2024-11-19T17:15:58,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:58,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/80dc36ed0b3f498ca41c2563d847d818 is 50, key is test_row_0/A:col10/1732036558835/Put/seqid=0 2024-11-19T17:15:58,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742386_1562 (size=14741) 2024-11-19T17:15:58,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:58,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036618893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:58,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:58,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:58,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:58,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036618894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:58,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:58,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:58,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036618899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:58,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:58,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:58,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:58,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:58,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:58,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:59,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:59,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036619014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:59,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:59,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036619014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:59,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:59,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:59,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:59,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:59,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036619014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:59,019 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:59,019 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:59,020 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:59,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:59,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:59,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:59,022 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/59c6a4bac5e045558a98308a772d4582 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/59c6a4bac5e045558a98308a772d4582 2024-11-19T17:15:59,026 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2d49416a7b2a16305b1ea9d2c71b4946/B of 2d49416a7b2a16305b1ea9d2c71b4946 into 59c6a4bac5e045558a98308a772d4582(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:15:59,026 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:59,026 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., storeName=2d49416a7b2a16305b1ea9d2c71b4946/B, priority=12, startTime=1732036558601; duration=0sec 2024-11-19T17:15:59,026 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:15:59,026 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d49416a7b2a16305b1ea9d2c71b4946:B 2024-11-19T17:15:59,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036619224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:59,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:59,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:59,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036619224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:59,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:59,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:59,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:59,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036619224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:59,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:59,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:59,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:59,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:59,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:59,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:59,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/80dc36ed0b3f498ca41c2563d847d818 2024-11-19T17:15:59,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/9e0f5b4f9f304a728ab901136ae72db2 is 50, key is test_row_0/B:col10/1732036558835/Put/seqid=0 2024-11-19T17:15:59,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742387_1563 (size=12301) 2024-11-19T17:15:59,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:59,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036619534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:59,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:59,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:59,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:59,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036619534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:59,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:59,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:15:59,539 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:59,539 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:59,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036619534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:15:59,539 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:59,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:59,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:15:59,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:15:59,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/9e0f5b4f9f304a728ab901136ae72db2 2024-11-19T17:15:59,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/b91668d8801c4b33847d27828436b3a2 is 50, key is test_row_0/C:col10/1732036558835/Put/seqid=0 2024-11-19T17:15:59,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742388_1564 (size=12301) 2024-11-19T17:15:59,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/b91668d8801c4b33847d27828436b3a2 2024-11-19T17:15:59,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/80dc36ed0b3f498ca41c2563d847d818 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/80dc36ed0b3f498ca41c2563d847d818 2024-11-19T17:15:59,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/80dc36ed0b3f498ca41c2563d847d818, entries=200, sequenceid=334, filesize=14.4 K 2024-11-19T17:15:59,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/9e0f5b4f9f304a728ab901136ae72db2 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/9e0f5b4f9f304a728ab901136ae72db2 2024-11-19T17:15:59,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/9e0f5b4f9f304a728ab901136ae72db2, entries=150, sequenceid=334, filesize=12.0 K 2024-11-19T17:15:59,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/b91668d8801c4b33847d27828436b3a2 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/b91668d8801c4b33847d27828436b3a2 2024-11-19T17:15:59,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/b91668d8801c4b33847d27828436b3a2, entries=150, sequenceid=334, filesize=12.0 K 2024-11-19T17:15:59,679 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 2d49416a7b2a16305b1ea9d2c71b4946 in 842ms, sequenceid=334, compaction requested=false 2024-11-19T17:15:59,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:15:59,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T17:15:59,684 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:15:59,684 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:15:59,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-19T17:15:59,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T17:15:59,685 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:15:59,686 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:15:59,686 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:15:59,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T17:15:59,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-19T17:15:59,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:15:59,839 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing 2d49416a7b2a16305b1ea9d2c71b4946 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T17:15:59,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=A 2024-11-19T17:15:59,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:59,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=B 2024-11-19T17:15:59,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:59,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=C 2024-11-19T17:15:59,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:15:59,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/7011db34373e464c9295e4944b603dee is 50, key is test_row_0/A:col10/1732036558893/Put/seqid=0 2024-11-19T17:15:59,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742389_1565 (size=12301) 2024-11-19T17:16:00,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T17:16:00,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:16:00,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:16:00,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:00,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036620065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:00,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:00,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036620066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:00,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:00,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:00,079 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:00,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036620071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:00,079 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:00,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:00,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:00,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:00,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:00,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036620183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:00,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:00,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:00,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:00,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036620183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:00,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:00,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:00,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:00,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:00,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:00,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036620184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:00,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:00,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:00,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:00,251 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/7011db34373e464c9295e4944b603dee 2024-11-19T17:16:00,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/3a4b2f2deb8f446aa821b2a6ad53c1f3 is 50, key is test_row_0/B:col10/1732036558893/Put/seqid=0 2024-11-19T17:16:00,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742390_1566 (size=12301) 2024-11-19T17:16:00,271 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/3a4b2f2deb8f446aa821b2a6ad53c1f3 2024-11-19T17:16:00,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/5d3c8d4378894f2e95372115b3eb54a6 is 50, key is test_row_0/C:col10/1732036558893/Put/seqid=0 2024-11-19T17:16:00,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742391_1567 (size=12301) 2024-11-19T17:16:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T17:16:00,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:00,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036620403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:00,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:00,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036620403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:00,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:00,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:00,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:00,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:00,414 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:00,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036620404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:00,414 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:00,414 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:00,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:00,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:00,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:00,685 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/5d3c8d4378894f2e95372115b3eb54a6 2024-11-19T17:16:00,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/7011db34373e464c9295e4944b603dee as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/7011db34373e464c9295e4944b603dee 2024-11-19T17:16:00,691 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/7011db34373e464c9295e4944b603dee, entries=150, sequenceid=355, filesize=12.0 K 2024-11-19T17:16:00,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/3a4b2f2deb8f446aa821b2a6ad53c1f3 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/3a4b2f2deb8f446aa821b2a6ad53c1f3 2024-11-19T17:16:00,695 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/3a4b2f2deb8f446aa821b2a6ad53c1f3, entries=150, sequenceid=355, filesize=12.0 K 2024-11-19T17:16:00,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/5d3c8d4378894f2e95372115b3eb54a6 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/5d3c8d4378894f2e95372115b3eb54a6 2024-11-19T17:16:00,699 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/5d3c8d4378894f2e95372115b3eb54a6, entries=150, sequenceid=355, filesize=12.0 K 2024-11-19T17:16:00,700 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 2d49416a7b2a16305b1ea9d2c71b4946 in 860ms, sequenceid=355, compaction requested=true 2024-11-19T17:16:00,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:16:00,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:16:00,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-19T17:16:00,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-19T17:16:00,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-19T17:16:00,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0150 sec 2024-11-19T17:16:00,704 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 1.0190 sec 2024-11-19T17:16:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:16:00,730 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d49416a7b2a16305b1ea9d2c71b4946 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T17:16:00,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=A 2024-11-19T17:16:00,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:00,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=B 2024-11-19T17:16:00,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:00,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=C 2024-11-19T17:16:00,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:00,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/ae073b22acf54a4295a974f91f22e7e9 is 50, key is test_row_0/A:col10/1732036560065/Put/seqid=0 2024-11-19T17:16:00,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742392_1568 (size=14741) 2024-11-19T17:16:00,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:00,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036620774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:00,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:00,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:00,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:00,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:00,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036620775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:00,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:00,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:00,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036620784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:00,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:00,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:00,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:00,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:00,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:00,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T17:16:00,823 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:16:00,824 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:16:00,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees 2024-11-19T17:16:00,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-19T17:16:00,826 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:16:00,826 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:16:00,826 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:16:00,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:00,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036620894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:00,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:00,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:00,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:00,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:00,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036620894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:00,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:00,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:00,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:00,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036620894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:00,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:00,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:00,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:00,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:00,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-19T17:16:00,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-19T17:16:00,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:16:00,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:16:00,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:16:00,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:00,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:00,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:01,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:01,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036621114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:01,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:01,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036621114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:01,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:01,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:01,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:01,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:01,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:01,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:01,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036621114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:01,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:01,125 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:01,125 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:01,125 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:01,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-19T17:16:01,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:16:01,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:16:01,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:16:01,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:01,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:01,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:01,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/ae073b22acf54a4295a974f91f22e7e9 2024-11-19T17:16:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-19T17:16:01,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/aa3efa7cbca44a04afb73307485a19c4 is 50, key is test_row_0/B:col10/1732036560065/Put/seqid=0 2024-11-19T17:16:01,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742393_1569 (size=12301) 2024-11-19T17:16:01,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/aa3efa7cbca44a04afb73307485a19c4 2024-11-19T17:16:01,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/cf4000da4abf4850b4df9012c9df3d12 is 50, key is test_row_0/C:col10/1732036560065/Put/seqid=0 2024-11-19T17:16:01,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742394_1570 (size=12301) 2024-11-19T17:16:01,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-19T17:16:01,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:16:01,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:16:01,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:16:01,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:01,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:01,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-19T17:16:01,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:16:01,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:16:01,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:16:01,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:01,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:01,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:01,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036621434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:01,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036621434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:01,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:01,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:01,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:01,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036621434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-19T17:16:01,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/cf4000da4abf4850b4df9012c9df3d12 2024-11-19T17:16:01,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/ae073b22acf54a4295a974f91f22e7e9 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/ae073b22acf54a4295a974f91f22e7e9 2024-11-19T17:16:01,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/ae073b22acf54a4295a974f91f22e7e9, entries=200, sequenceid=368, filesize=14.4 K 2024-11-19T17:16:01,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/aa3efa7cbca44a04afb73307485a19c4 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/aa3efa7cbca44a04afb73307485a19c4 2024-11-19T17:16:01,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/aa3efa7cbca44a04afb73307485a19c4, entries=150, sequenceid=368, filesize=12.0 K 2024-11-19T17:16:01,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/cf4000da4abf4850b4df9012c9df3d12 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/cf4000da4abf4850b4df9012c9df3d12 2024-11-19T17:16:01,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/cf4000da4abf4850b4df9012c9df3d12, entries=150, sequenceid=368, filesize=12.0 K 2024-11-19T17:16:01,575 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 2d49416a7b2a16305b1ea9d2c71b4946 in 845ms, sequenceid=368, compaction requested=true 2024-11-19T17:16:01,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:16:01,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d49416a7b2a16305b1ea9d2c71b4946:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:16:01,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:01,575 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:16:01,575 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:16:01,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d49416a7b2a16305b1ea9d2c71b4946:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:16:01,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:01,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d49416a7b2a16305b1ea9d2c71b4946:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:16:01,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:16:01,577 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:16:01,577 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54800 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:16:01,577 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 2d49416a7b2a16305b1ea9d2c71b4946/A is initiating minor compaction (all files) 2024-11-19T17:16:01,577 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 2d49416a7b2a16305b1ea9d2c71b4946/B is initiating minor compaction (all files) 2024-11-19T17:16:01,577 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d49416a7b2a16305b1ea9d2c71b4946/A in TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:16:01,577 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d49416a7b2a16305b1ea9d2c71b4946/B in TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:16:01,577 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/59c6a4bac5e045558a98308a772d4582, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/9e0f5b4f9f304a728ab901136ae72db2, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/3a4b2f2deb8f446aa821b2a6ad53c1f3, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/aa3efa7cbca44a04afb73307485a19c4] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp, totalSize=48.8 K 2024-11-19T17:16:01,577 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/1ae26d7fb4174c85b75807da00c6ee68, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/80dc36ed0b3f498ca41c2563d847d818, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/7011db34373e464c9295e4944b603dee, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/ae073b22acf54a4295a974f91f22e7e9] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp, totalSize=53.5 K 2024-11-19T17:16:01,577 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 59c6a4bac5e045558a98308a772d4582, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732036556950 2024-11-19T17:16:01,577 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ae26d7fb4174c85b75807da00c6ee68, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732036556950 2024-11-19T17:16:01,578 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80dc36ed0b3f498ca41c2563d847d818, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732036558164 2024-11-19T17:16:01,578 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e0f5b4f9f304a728ab901136ae72db2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732036558164 2024-11-19T17:16:01,578 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7011db34373e464c9295e4944b603dee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732036558892 2024-11-19T17:16:01,578 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a4b2f2deb8f446aa821b2a6ad53c1f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732036558892 2024-11-19T17:16:01,578 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae073b22acf54a4295a974f91f22e7e9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732036560064 2024-11-19T17:16:01,578 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting aa3efa7cbca44a04afb73307485a19c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732036560065 2024-11-19T17:16:01,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-19T17:16:01,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:16:01,590 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2902): Flushing 2d49416a7b2a16305b1ea9d2c71b4946 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T17:16:01,590 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d49416a7b2a16305b1ea9d2c71b4946#B#compaction#487 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:16:01,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=A 2024-11-19T17:16:01,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:01,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=B 2024-11-19T17:16:01,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:01,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=C 2024-11-19T17:16:01,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:01,591 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/bebfe8392a7545eda8da691c3bd9c100 is 50, key is test_row_0/B:col10/1732036560065/Put/seqid=0 2024-11-19T17:16:01,594 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d49416a7b2a16305b1ea9d2c71b4946#A#compaction#488 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:16:01,595 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/65957afacc6f41a0aea7489a74d4e1c7 is 50, key is test_row_0/A:col10/1732036560065/Put/seqid=0 2024-11-19T17:16:01,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/fcf3959291bc4090a452e333b89f5aab is 50, key is test_row_0/A:col10/1732036560768/Put/seqid=0 2024-11-19T17:16:01,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742396_1572 (size=13153) 2024-11-19T17:16:01,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742395_1571 (size=13153) 2024-11-19T17:16:01,616 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/bebfe8392a7545eda8da691c3bd9c100 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/bebfe8392a7545eda8da691c3bd9c100 2024-11-19T17:16:01,618 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/65957afacc6f41a0aea7489a74d4e1c7 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/65957afacc6f41a0aea7489a74d4e1c7 2024-11-19T17:16:01,621 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2d49416a7b2a16305b1ea9d2c71b4946/B of 2d49416a7b2a16305b1ea9d2c71b4946 into bebfe8392a7545eda8da691c3bd9c100(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:16:01,621 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:16:01,621 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., storeName=2d49416a7b2a16305b1ea9d2c71b4946/B, priority=12, startTime=1732036561575; duration=0sec 2024-11-19T17:16:01,621 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:16:01,621 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d49416a7b2a16305b1ea9d2c71b4946:B 2024-11-19T17:16:01,622 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:16:01,623 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:16:01,623 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 2d49416a7b2a16305b1ea9d2c71b4946/C is initiating minor compaction (all files) 2024-11-19T17:16:01,624 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d49416a7b2a16305b1ea9d2c71b4946/C in TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:16:01,624 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/af254ce784584828b33832d5c44cd801, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/b91668d8801c4b33847d27828436b3a2, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/5d3c8d4378894f2e95372115b3eb54a6, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/cf4000da4abf4850b4df9012c9df3d12] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp, totalSize=48.8 K 2024-11-19T17:16:01,624 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting af254ce784584828b33832d5c44cd801, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732036556950 2024-11-19T17:16:01,625 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting b91668d8801c4b33847d27828436b3a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732036558164 2024-11-19T17:16:01,625 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d3c8d4378894f2e95372115b3eb54a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732036558892 2024-11-19T17:16:01,625 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting cf4000da4abf4850b4df9012c9df3d12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732036560065 2024-11-19T17:16:01,628 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2d49416a7b2a16305b1ea9d2c71b4946/A of 2d49416a7b2a16305b1ea9d2c71b4946 into 65957afacc6f41a0aea7489a74d4e1c7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:16:01,628 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:16:01,628 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., storeName=2d49416a7b2a16305b1ea9d2c71b4946/A, priority=12, startTime=1732036561575; duration=0sec 2024-11-19T17:16:01,628 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:01,628 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d49416a7b2a16305b1ea9d2c71b4946:A 2024-11-19T17:16:01,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742397_1573 (size=12301) 2024-11-19T17:16:01,630 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/fcf3959291bc4090a452e333b89f5aab 2024-11-19T17:16:01,639 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d49416a7b2a16305b1ea9d2c71b4946#C#compaction#490 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:16:01,640 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/e152bb719ab848faa599155301de30a9 is 50, key is test_row_0/C:col10/1732036560065/Put/seqid=0 2024-11-19T17:16:01,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/10cd31bc4eff4d5b9592dd146ee10a56 is 50, key is test_row_0/B:col10/1732036560768/Put/seqid=0 2024-11-19T17:16:01,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742398_1574 (size=13153) 2024-11-19T17:16:01,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742399_1575 (size=12301) 2024-11-19T17:16:01,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:16:01,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:16:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-19T17:16:01,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:01,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036621974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:01,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:01,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:01,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:01,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:01,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036621976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:01,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:01,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:01,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:01,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:01,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036621987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:01,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:01,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:01,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:02,057 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/10cd31bc4eff4d5b9592dd146ee10a56 2024-11-19T17:16:02,059 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/e152bb719ab848faa599155301de30a9 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/e152bb719ab848faa599155301de30a9 2024-11-19T17:16:02,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/9ce66764de38421da63dbd34d84e6b5d is 50, key is test_row_0/C:col10/1732036560768/Put/seqid=0 2024-11-19T17:16:02,064 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2d49416a7b2a16305b1ea9d2c71b4946/C of 2d49416a7b2a16305b1ea9d2c71b4946 into e152bb719ab848faa599155301de30a9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:16:02,064 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:16:02,064 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., storeName=2d49416a7b2a16305b1ea9d2c71b4946/C, priority=12, startTime=1732036561576; duration=0sec 2024-11-19T17:16:02,064 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:02,064 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d49416a7b2a16305b1ea9d2c71b4946:C 2024-11-19T17:16:02,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742400_1576 (size=12301) 2024-11-19T17:16:02,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:02,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036622093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:02,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:02,095 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:02,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036622093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:02,095 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:02,095 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:02,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:02,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:02,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:02,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:02,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036622103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:02,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:02,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:02,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:02,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:02,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036622304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:02,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:02,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:02,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036622304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:02,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:02,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:02,307 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:02,307 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:02,307 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:02,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:02,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036622313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:02,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:02,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:02,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:02,467 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/9ce66764de38421da63dbd34d84e6b5d 2024-11-19T17:16:02,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/fcf3959291bc4090a452e333b89f5aab as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/fcf3959291bc4090a452e333b89f5aab 2024-11-19T17:16:02,475 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/fcf3959291bc4090a452e333b89f5aab, entries=150, sequenceid=387, filesize=12.0 K 2024-11-19T17:16:02,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/10cd31bc4eff4d5b9592dd146ee10a56 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/10cd31bc4eff4d5b9592dd146ee10a56 2024-11-19T17:16:02,478 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/10cd31bc4eff4d5b9592dd146ee10a56, entries=150, sequenceid=387, filesize=12.0 K 2024-11-19T17:16:02,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/9ce66764de38421da63dbd34d84e6b5d as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/9ce66764de38421da63dbd34d84e6b5d 2024-11-19T17:16:02,482 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/9ce66764de38421da63dbd34d84e6b5d, entries=150, sequenceid=387, filesize=12.0 K 2024-11-19T17:16:02,483 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 2d49416a7b2a16305b1ea9d2c71b4946 in 893ms, sequenceid=387, compaction requested=false 2024-11-19T17:16:02,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2603): Flush status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:16:02,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:16:02,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-11-19T17:16:02,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=144 2024-11-19T17:16:02,485 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-19T17:16:02,486 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6580 sec 2024-11-19T17:16:02,487 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees in 1.6620 sec 2024-11-19T17:16:02,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:16:02,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d49416a7b2a16305b1ea9d2c71b4946 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T17:16:02,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=A 2024-11-19T17:16:02,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:02,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=B 2024-11-19T17:16:02,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:02,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=C 2024-11-19T17:16:02,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:02,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/032013f72d0d41ab94abef6ff5244390 is 50, key is test_row_0/A:col10/1732036561961/Put/seqid=0 2024-11-19T17:16:02,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742401_1577 (size=14741) 2024-11-19T17:16:02,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/032013f72d0d41ab94abef6ff5244390 2024-11-19T17:16:02,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/16d5e941dbd1496daeb5bca2c81a553b is 50, key is test_row_0/B:col10/1732036561961/Put/seqid=0 2024-11-19T17:16:02,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742402_1578 (size=12301) 2024-11-19T17:16:02,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/16d5e941dbd1496daeb5bca2c81a553b 2024-11-19T17:16:02,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/0f236016d7b745e9b0a44b5bfbf91e5b is 50, key is test_row_0/C:col10/1732036561961/Put/seqid=0 2024-11-19T17:16:02,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742403_1579 (size=12301) 2024-11-19T17:16:02,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:02,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036622662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:02,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:02,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:02,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:02,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:02,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036622663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:02,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:02,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:02,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:02,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:02,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036622664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:02,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:02,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:02,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:02,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:02,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036622774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:02,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:02,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:02,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:02,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036622783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:02,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036622783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:02,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:02,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:02,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:02,793 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:02,793 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:02,793 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:02,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-19T17:16:02,974 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:16:02,975 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:16:02,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees 2024-11-19T17:16:02,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-19T17:16:02,976 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:16:02,977 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:16:02,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:16:02,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:02,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036622984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:02,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:02,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:02,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:03,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:03,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036623003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:03,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:03,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:03,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036623003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:03,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:03,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:03,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:03,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:03,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:03,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/0f236016d7b745e9b0a44b5bfbf91e5b 2024-11-19T17:16:03,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/032013f72d0d41ab94abef6ff5244390 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/032013f72d0d41ab94abef6ff5244390 2024-11-19T17:16:03,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/032013f72d0d41ab94abef6ff5244390, entries=200, sequenceid=404, filesize=14.4 K 2024-11-19T17:16:03,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/16d5e941dbd1496daeb5bca2c81a553b as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/16d5e941dbd1496daeb5bca2c81a553b 2024-11-19T17:16:03,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/16d5e941dbd1496daeb5bca2c81a553b, entries=150, sequenceid=404, filesize=12.0 K 2024-11-19T17:16:03,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/0f236016d7b745e9b0a44b5bfbf91e5b as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/0f236016d7b745e9b0a44b5bfbf91e5b 2024-11-19T17:16:03,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/0f236016d7b745e9b0a44b5bfbf91e5b, entries=150, sequenceid=404, filesize=12.0 K 2024-11-19T17:16:03,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 2d49416a7b2a16305b1ea9d2c71b4946 in 444ms, sequenceid=404, compaction requested=true 2024-11-19T17:16:03,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:16:03,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d49416a7b2a16305b1ea9d2c71b4946:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:16:03,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:03,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d49416a7b2a16305b1ea9d2c71b4946:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:16:03,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:03,061 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:16:03,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d49416a7b2a16305b1ea9d2c71b4946:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:16:03,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:16:03,062 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:16:03,062 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:16:03,062 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:16:03,062 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): 2d49416a7b2a16305b1ea9d2c71b4946/A is initiating minor compaction (all files) 2024-11-19T17:16:03,062 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 2d49416a7b2a16305b1ea9d2c71b4946/B is initiating minor compaction (all files) 2024-11-19T17:16:03,062 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d49416a7b2a16305b1ea9d2c71b4946/A in TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:16:03,062 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d49416a7b2a16305b1ea9d2c71b4946/B in TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:16:03,063 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/65957afacc6f41a0aea7489a74d4e1c7, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/fcf3959291bc4090a452e333b89f5aab, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/032013f72d0d41ab94abef6ff5244390] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp, totalSize=39.3 K 2024-11-19T17:16:03,063 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/bebfe8392a7545eda8da691c3bd9c100, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/10cd31bc4eff4d5b9592dd146ee10a56, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/16d5e941dbd1496daeb5bca2c81a553b] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp, totalSize=36.9 K 2024-11-19T17:16:03,063 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65957afacc6f41a0aea7489a74d4e1c7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732036560065 2024-11-19T17:16:03,063 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting bebfe8392a7545eda8da691c3bd9c100, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732036560065 2024-11-19T17:16:03,063 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcf3959291bc4090a452e333b89f5aab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1732036560768 2024-11-19T17:16:03,063 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 10cd31bc4eff4d5b9592dd146ee10a56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1732036560768 2024-11-19T17:16:03,063 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 032013f72d0d41ab94abef6ff5244390, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732036561961 2024-11-19T17:16:03,063 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 16d5e941dbd1496daeb5bca2c81a553b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732036561961 2024-11-19T17:16:03,069 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d49416a7b2a16305b1ea9d2c71b4946#B#compaction#497 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:16:03,070 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/6ff45f5ce3344ccdb609c30c73b6d787 is 50, key is test_row_0/B:col10/1732036561961/Put/seqid=0 2024-11-19T17:16:03,070 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d49416a7b2a16305b1ea9d2c71b4946#A#compaction#496 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:16:03,070 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/69de6df2dfa14c2791cff560e4d1db4c is 50, key is test_row_0/A:col10/1732036561961/Put/seqid=0 2024-11-19T17:16:03,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742404_1580 (size=13255) 2024-11-19T17:16:03,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742405_1581 (size=13255) 2024-11-19T17:16:03,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-19T17:16:03,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-19T17:16:03,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:16:03,130 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2902): Flushing 2d49416a7b2a16305b1ea9d2c71b4946 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T17:16:03,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=A 2024-11-19T17:16:03,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:03,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=B 2024-11-19T17:16:03,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:03,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=C 2024-11-19T17:16:03,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:03,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/0085815c2edd4a1ba8b560e22d374af4 is 50, key is test_row_0/A:col10/1732036562637/Put/seqid=0 2024-11-19T17:16:03,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742406_1582 (size=12301) 2024-11-19T17:16:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-19T17:16:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:16:03,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. as already flushing 2024-11-19T17:16:03,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036623349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:03,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036623354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:03,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:03,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036623355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:03,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:03,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036623474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:03,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036623474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:03,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:03,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:03,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:03,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036623474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:03,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:03,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:03,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:03,479 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:03,479 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:03,479 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/6ff45f5ce3344ccdb609c30c73b6d787 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/6ff45f5ce3344ccdb609c30c73b6d787 2024-11-19T17:16:03,479 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:03,479 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/69de6df2dfa14c2791cff560e4d1db4c as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/69de6df2dfa14c2791cff560e4d1db4c 2024-11-19T17:16:03,483 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d49416a7b2a16305b1ea9d2c71b4946/B of 2d49416a7b2a16305b1ea9d2c71b4946 into 6ff45f5ce3344ccdb609c30c73b6d787(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:16:03,483 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:16:03,483 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., storeName=2d49416a7b2a16305b1ea9d2c71b4946/B, priority=13, startTime=1732036563061; duration=0sec 2024-11-19T17:16:03,483 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:16:03,483 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d49416a7b2a16305b1ea9d2c71b4946:B 2024-11-19T17:16:03,483 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:16:03,484 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d49416a7b2a16305b1ea9d2c71b4946/A of 2d49416a7b2a16305b1ea9d2c71b4946 into 69de6df2dfa14c2791cff560e4d1db4c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:16:03,484 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:16:03,484 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., storeName=2d49416a7b2a16305b1ea9d2c71b4946/A, priority=13, startTime=1732036563061; duration=0sec 2024-11-19T17:16:03,484 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:03,484 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d49416a7b2a16305b1ea9d2c71b4946:A 2024-11-19T17:16:03,485 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:16:03,485 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): 2d49416a7b2a16305b1ea9d2c71b4946/C is initiating minor compaction (all files) 2024-11-19T17:16:03,485 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d49416a7b2a16305b1ea9d2c71b4946/C in TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:16:03,485 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/e152bb719ab848faa599155301de30a9, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/9ce66764de38421da63dbd34d84e6b5d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/0f236016d7b745e9b0a44b5bfbf91e5b] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp, totalSize=36.9 K 2024-11-19T17:16:03,485 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting e152bb719ab848faa599155301de30a9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732036560065 2024-11-19T17:16:03,485 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ce66764de38421da63dbd34d84e6b5d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1732036560768 2024-11-19T17:16:03,486 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f236016d7b745e9b0a44b5bfbf91e5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732036561961 2024-11-19T17:16:03,493 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d49416a7b2a16305b1ea9d2c71b4946#C#compaction#499 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:16:03,494 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/7843760307be49d29db25c279e8cb6f6 is 50, key is test_row_0/C:col10/1732036561961/Put/seqid=0 2024-11-19T17:16:03,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742407_1583 (size=13255) 2024-11-19T17:16:03,505 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/7843760307be49d29db25c279e8cb6f6 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/7843760307be49d29db25c279e8cb6f6 2024-11-19T17:16:03,510 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d49416a7b2a16305b1ea9d2c71b4946/C of 2d49416a7b2a16305b1ea9d2c71b4946 into 7843760307be49d29db25c279e8cb6f6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:16:03,510 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:16:03,510 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., storeName=2d49416a7b2a16305b1ea9d2c71b4946/C, priority=13, startTime=1732036563061; duration=0sec 2024-11-19T17:16:03,511 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:03,511 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d49416a7b2a16305b1ea9d2c71b4946:C 2024-11-19T17:16:03,540 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/0085815c2edd4a1ba8b560e22d374af4 2024-11-19T17:16:03,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/f40318fb38f34d4eb4b5dcbe75a079cc is 50, key is test_row_0/B:col10/1732036562637/Put/seqid=0 2024-11-19T17:16:03,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742408_1584 (size=12301) 2024-11-19T17:16:03,558 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/f40318fb38f34d4eb4b5dcbe75a079cc 2024-11-19T17:16:03,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/dcd2fbbd92344b8cb01007d8b35ed35e is 50, key is test_row_0/C:col10/1732036562637/Put/seqid=0 2024-11-19T17:16:03,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742409_1585 (size=12301) 2024-11-19T17:16:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-19T17:16:03,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51532 deadline: 1732036623684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:03,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51560 deadline: 1732036623684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:03,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:03,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:03,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:03,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51602 deadline: 1732036623684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:03,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:03,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:03,695 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 , the old value is region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:03,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:03,695 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:03,695 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., hostname=280a42d09ec8,33873,1732036422473, seqNum=2 because the exception is null or not the one we care about 2024-11-19T17:16:03,775 INFO [Thread-2211 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2211. 2024-11-19T17:16:03,775 DEBUG [Thread-2211 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:16:03,775 DEBUG [Thread-2211 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:03,775 DEBUG [Thread-2211 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:03,775 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:16:03,777 INFO [Thread-2215 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2215. 2024-11-19T17:16:03,777 DEBUG [Thread-2215 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:16:03,777 DEBUG [Thread-2215 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:03,777 DEBUG [Thread-2215 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:03,777 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:16:03,781 INFO [Thread-2209 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2209. 2024-11-19T17:16:03,781 DEBUG [Thread-2209 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:16:03,781 DEBUG [Thread-2209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:03,781 DEBUG [Thread-2209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:03,781 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:16:03,785 INFO [Thread-2213 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2213. 2024-11-19T17:16:03,785 DEBUG [Thread-2213 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:16:03,785 DEBUG [Thread-2213 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:03,785 DEBUG [Thread-2213 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:03,785 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:16:03,786 INFO [Thread-2207 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2207. 2024-11-19T17:16:03,786 DEBUG [Thread-2207 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:16:03,786 DEBUG [Thread-2207 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:03,786 DEBUG [Thread-2207 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:03,786 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:16:03,974 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/dcd2fbbd92344b8cb01007d8b35ed35e 2024-11-19T17:16:03,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/0085815c2edd4a1ba8b560e22d374af4 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/0085815c2edd4a1ba8b560e22d374af4 2024-11-19T17:16:03,980 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/0085815c2edd4a1ba8b560e22d374af4, entries=150, sequenceid=422, filesize=12.0 K 2024-11-19T17:16:03,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/f40318fb38f34d4eb4b5dcbe75a079cc as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/f40318fb38f34d4eb4b5dcbe75a079cc 2024-11-19T17:16:03,982 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/f40318fb38f34d4eb4b5dcbe75a079cc, entries=150, sequenceid=422, filesize=12.0 K 2024-11-19T17:16:03,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/dcd2fbbd92344b8cb01007d8b35ed35e as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/dcd2fbbd92344b8cb01007d8b35ed35e 2024-11-19T17:16:03,985 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/dcd2fbbd92344b8cb01007d8b35ed35e, entries=150, sequenceid=422, filesize=12.0 K 2024-11-19T17:16:03,986 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 2d49416a7b2a16305b1ea9d2c71b4946 in 857ms, sequenceid=422, compaction requested=false 2024-11-19T17:16:03,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2603): Flush status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:16:03,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:16:03,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=146 2024-11-19T17:16:03,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=146 2024-11-19T17:16:03,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-19T17:16:03,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0100 sec 2024-11-19T17:16:03,990 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees in 1.0140 sec 2024-11-19T17:16:04,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:16:04,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d49416a7b2a16305b1ea9d2c71b4946 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T17:16:04,006 INFO [Thread-2204 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2204. 2024-11-19T17:16:04,006 INFO [Thread-2198 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2198. 2024-11-19T17:16:04,006 DEBUG [Thread-2204 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:16:04,006 DEBUG [Thread-2198 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:16:04,006 DEBUG [Thread-2204 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:04,006 DEBUG [Thread-2198 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:04,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=A 2024-11-19T17:16:04,006 INFO [Thread-2200 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2200. 2024-11-19T17:16:04,006 DEBUG [Thread-2200 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:16:04,006 DEBUG [Thread-2204 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:04,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:04,006 DEBUG [Thread-2198 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:04,006 DEBUG [Thread-2200 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:04,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=B 2024-11-19T17:16:04,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:04,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=C 2024-11-19T17:16:04,006 DEBUG [Thread-2200 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:04,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:04,006 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:16:04,006 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:16:04,006 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:16:04,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/776dc0eb845e4b6992255abe6b90d41a is 50, key is test_row_0/A:col10/1732036564004/Put/seqid=0 2024-11-19T17:16:04,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742410_1586 (size=12301) 2024-11-19T17:16:04,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-19T17:16:04,114 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:16:04,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/776dc0eb845e4b6992255abe6b90d41a 2024-11-19T17:16:04,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/bdbb254438034753a1d0ec80a4305177 is 50, key is test_row_0/B:col10/1732036564004/Put/seqid=0 2024-11-19T17:16:04,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742411_1587 (size=12301) 2024-11-19T17:16:04,735 INFO [Thread-2202 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2202. 2024-11-19T17:16:04,735 DEBUG [Thread-2202 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:16:04,735 DEBUG [Thread-2202 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:04,736 DEBUG [Thread-2202 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:04,736 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:16:04,744 INFO [Thread-2196 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2196. 2024-11-19T17:16:04,744 DEBUG [Thread-2196 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:16:04,744 DEBUG [Thread-2196 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:04,745 DEBUG [Thread-2196 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:04,745 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:16:04,745 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T17:16:04,745 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-19T17:16:04,745 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-11-19T17:16:04,745 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-11-19T17:16:04,745 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-19T17:16:04,745 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-11-19T17:16:04,745 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T17:16:04,745 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T17:16:04,745 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1101 2024-11-19T17:16:04,745 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3299 rows 2024-11-19T17:16:04,745 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1107 2024-11-19T17:16:04,745 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3320 rows 2024-11-19T17:16:04,745 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1104 2024-11-19T17:16:04,745 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3306 rows 2024-11-19T17:16:04,745 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1104 2024-11-19T17:16:04,745 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3309 rows 2024-11-19T17:16:04,745 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1106 2024-11-19T17:16:04,745 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3317 rows 2024-11-19T17:16:04,745 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T17:16:04,745 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T17:16:04,745 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:04,745 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:04,745 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:16:04,746 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T17:16:04,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T17:16:04,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-19T17:16:04,749 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732036564749"}]},"ts":"1732036564749"} 2024-11-19T17:16:04,750 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T17:16:04,750 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T17:16:04,750 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T17:16:04,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d49416a7b2a16305b1ea9d2c71b4946, UNASSIGN}] 2024-11-19T17:16:04,752 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d49416a7b2a16305b1ea9d2c71b4946, UNASSIGN 2024-11-19T17:16:04,752 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=2d49416a7b2a16305b1ea9d2c71b4946, regionState=CLOSING, regionLocation=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:04,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d49416a7b2a16305b1ea9d2c71b4946, UNASSIGN because future has completed 2024-11-19T17:16:04,754 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T17:16:04,754 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473}] 2024-11-19T17:16:04,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/bdbb254438034753a1d0ec80a4305177 2024-11-19T17:16:04,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/9d4073978651455894cbb8dfd03cd45d is 50, key is test_row_0/C:col10/1732036564004/Put/seqid=0 2024-11-19T17:16:04,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742412_1588 (size=12301) 2024-11-19T17:16:04,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-19T17:16:04,905 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:16:04,906 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T17:16:04,906 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing 2d49416a7b2a16305b1ea9d2c71b4946, disabling compactions & flushes 2024-11-19T17:16:04,906 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:16:05,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-19T17:16:05,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/9d4073978651455894cbb8dfd03cd45d 2024-11-19T17:16:05,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/776dc0eb845e4b6992255abe6b90d41a as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/776dc0eb845e4b6992255abe6b90d41a 2024-11-19T17:16:05,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/776dc0eb845e4b6992255abe6b90d41a, entries=150, sequenceid=442, filesize=12.0 K 2024-11-19T17:16:05,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/bdbb254438034753a1d0ec80a4305177 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/bdbb254438034753a1d0ec80a4305177 2024-11-19T17:16:05,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/bdbb254438034753a1d0ec80a4305177, entries=150, sequenceid=442, filesize=12.0 K 2024-11-19T17:16:05,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/9d4073978651455894cbb8dfd03cd45d as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/9d4073978651455894cbb8dfd03cd45d 2024-11-19T17:16:05,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/9d4073978651455894cbb8dfd03cd45d, entries=150, sequenceid=442, filesize=12.0 K 2024-11-19T17:16:05,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=13.42 KB/13740 for 2d49416a7b2a16305b1ea9d2c71b4946 in 1235ms, sequenceid=442, compaction requested=true 2024-11-19T17:16:05,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d49416a7b2a16305b1ea9d2c71b4946: 2024-11-19T17:16:05,241 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:16:05,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d49416a7b2a16305b1ea9d2c71b4946:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:16:05,241 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:16:05,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:05,241 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. after waiting 0 ms 2024-11-19T17:16:05,241 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. because compaction request was cancelled 2024-11-19T17:16:05,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d49416a7b2a16305b1ea9d2c71b4946:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:16:05,241 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:16:05,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:05,241 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d49416a7b2a16305b1ea9d2c71b4946:A 2024-11-19T17:16:05,241 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. because compaction request was cancelled 2024-11-19T17:16:05,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d49416a7b2a16305b1ea9d2c71b4946:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:16:05,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:16:05,241 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d49416a7b2a16305b1ea9d2c71b4946:B 2024-11-19T17:16:05,241 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. because compaction request was cancelled 2024-11-19T17:16:05,241 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d49416a7b2a16305b1ea9d2c71b4946:C 2024-11-19T17:16:05,241 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(2902): Flushing 2d49416a7b2a16305b1ea9d2c71b4946 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-19T17:16:05,241 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=A 2024-11-19T17:16:05,242 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:05,242 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=B 2024-11-19T17:16:05,242 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:05,242 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2d49416a7b2a16305b1ea9d2c71b4946, store=C 2024-11-19T17:16:05,242 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:05,244 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/ad87416562dc4db3bcfc204fd5c45022 is 50, key is test_row_1/A:col10/1732036564743/Put/seqid=0 2024-11-19T17:16:05,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742413_1589 (size=9857) 2024-11-19T17:16:05,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-19T17:16:05,648 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/ad87416562dc4db3bcfc204fd5c45022 2024-11-19T17:16:05,653 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/3179271f756347a4bf923fd94f0a0104 is 50, key is test_row_1/B:col10/1732036564743/Put/seqid=0 2024-11-19T17:16:05,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742414_1590 (size=9857) 2024-11-19T17:16:05,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-19T17:16:06,056 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/3179271f756347a4bf923fd94f0a0104 2024-11-19T17:16:06,061 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/236c2b3826b54e9aaffcaec001b16a0f is 50, key is test_row_1/C:col10/1732036564743/Put/seqid=0 2024-11-19T17:16:06,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742415_1591 (size=9857) 2024-11-19T17:16:06,464 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/236c2b3826b54e9aaffcaec001b16a0f 2024-11-19T17:16:06,468 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/A/ad87416562dc4db3bcfc204fd5c45022 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/ad87416562dc4db3bcfc204fd5c45022 2024-11-19T17:16:06,470 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/ad87416562dc4db3bcfc204fd5c45022, entries=100, sequenceid=447, filesize=9.6 K 2024-11-19T17:16:06,470 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/B/3179271f756347a4bf923fd94f0a0104 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/3179271f756347a4bf923fd94f0a0104 2024-11-19T17:16:06,473 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/3179271f756347a4bf923fd94f0a0104, entries=100, sequenceid=447, filesize=9.6 K 2024-11-19T17:16:06,473 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/.tmp/C/236c2b3826b54e9aaffcaec001b16a0f as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/236c2b3826b54e9aaffcaec001b16a0f 2024-11-19T17:16:06,476 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/236c2b3826b54e9aaffcaec001b16a0f, entries=100, sequenceid=447, filesize=9.6 K 2024-11-19T17:16:06,476 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 2d49416a7b2a16305b1ea9d2c71b4946 in 1235ms, sequenceid=447, compaction requested=true 2024-11-19T17:16:06,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/e39149d3b4a34687946f4eb6079a35f8, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/95e85861fa7041d0b5569eac7a900f29, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/fe89bc1d82da44c7a7ee4a92e00ed5f5, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/09fde3fb2a63424a890f02f82f57938e, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/2141df3479a74e0c91df5bfa77a4a136, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/e3bae5df23c34a6d8b86e75998b438dc, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/9951c95ca7064ebea6a5fe96d8d02779, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/36257bb45aaf4632be48d9a543639a37, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/5b004a0dad354150b6c9d9ae886f5a98, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/48ed4bc8ca114150b07c562ef3f3dc15, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/2a3abf5f4c7c41da8574c3487ce0fae4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/56a8e67f27994be798086497b67e4f8c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/a024492f5a0c44779900b09749873058, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/1b91af9dd89d4e099d55cec1357ae453, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/eaf1b21566a949d29f9517a658328387, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/8421315e1d2e4bb591d05f4f350825aa, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/57a3da48c0044894a172ddc4a94855ac, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/e501b554672c4277919a20b6640d0513, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/7ab35b3c046a43c5b30217f87b262cdd, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/33b784943c114525aec7f6713d995f13, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/d03adaa7770b4c6a995c42f5cb0f53ce, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/1ae26d7fb4174c85b75807da00c6ee68, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/80dc36ed0b3f498ca41c2563d847d818, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/7011db34373e464c9295e4944b603dee, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/ae073b22acf54a4295a974f91f22e7e9, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/65957afacc6f41a0aea7489a74d4e1c7, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/fcf3959291bc4090a452e333b89f5aab, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/032013f72d0d41ab94abef6ff5244390] to archive 2024-11-19T17:16:06,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:16:06,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/e39149d3b4a34687946f4eb6079a35f8 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/e39149d3b4a34687946f4eb6079a35f8 2024-11-19T17:16:06,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/95e85861fa7041d0b5569eac7a900f29 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/95e85861fa7041d0b5569eac7a900f29 2024-11-19T17:16:06,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/fe89bc1d82da44c7a7ee4a92e00ed5f5 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/fe89bc1d82da44c7a7ee4a92e00ed5f5 2024-11-19T17:16:06,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/09fde3fb2a63424a890f02f82f57938e to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/09fde3fb2a63424a890f02f82f57938e 2024-11-19T17:16:06,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/2141df3479a74e0c91df5bfa77a4a136 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/2141df3479a74e0c91df5bfa77a4a136 2024-11-19T17:16:06,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/e3bae5df23c34a6d8b86e75998b438dc to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/e3bae5df23c34a6d8b86e75998b438dc 2024-11-19T17:16:06,483 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/9951c95ca7064ebea6a5fe96d8d02779 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/9951c95ca7064ebea6a5fe96d8d02779 2024-11-19T17:16:06,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/36257bb45aaf4632be48d9a543639a37 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/36257bb45aaf4632be48d9a543639a37 2024-11-19T17:16:06,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/5b004a0dad354150b6c9d9ae886f5a98 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/5b004a0dad354150b6c9d9ae886f5a98 2024-11-19T17:16:06,486 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/48ed4bc8ca114150b07c562ef3f3dc15 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/48ed4bc8ca114150b07c562ef3f3dc15 2024-11-19T17:16:06,486 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/2a3abf5f4c7c41da8574c3487ce0fae4 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/2a3abf5f4c7c41da8574c3487ce0fae4 2024-11-19T17:16:06,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/56a8e67f27994be798086497b67e4f8c to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/56a8e67f27994be798086497b67e4f8c 2024-11-19T17:16:06,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/a024492f5a0c44779900b09749873058 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/a024492f5a0c44779900b09749873058 2024-11-19T17:16:06,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/1b91af9dd89d4e099d55cec1357ae453 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/1b91af9dd89d4e099d55cec1357ae453 2024-11-19T17:16:06,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/eaf1b21566a949d29f9517a658328387 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/eaf1b21566a949d29f9517a658328387 2024-11-19T17:16:06,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/8421315e1d2e4bb591d05f4f350825aa to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/8421315e1d2e4bb591d05f4f350825aa 2024-11-19T17:16:06,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/57a3da48c0044894a172ddc4a94855ac to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/57a3da48c0044894a172ddc4a94855ac 2024-11-19T17:16:06,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/e501b554672c4277919a20b6640d0513 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/e501b554672c4277919a20b6640d0513 2024-11-19T17:16:06,492 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/7ab35b3c046a43c5b30217f87b262cdd to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/7ab35b3c046a43c5b30217f87b262cdd 2024-11-19T17:16:06,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/33b784943c114525aec7f6713d995f13 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/33b784943c114525aec7f6713d995f13 2024-11-19T17:16:06,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/d03adaa7770b4c6a995c42f5cb0f53ce to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/d03adaa7770b4c6a995c42f5cb0f53ce 2024-11-19T17:16:06,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/1ae26d7fb4174c85b75807da00c6ee68 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/1ae26d7fb4174c85b75807da00c6ee68 2024-11-19T17:16:06,495 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/80dc36ed0b3f498ca41c2563d847d818 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/80dc36ed0b3f498ca41c2563d847d818 2024-11-19T17:16:06,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/7011db34373e464c9295e4944b603dee to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/7011db34373e464c9295e4944b603dee 2024-11-19T17:16:06,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/ae073b22acf54a4295a974f91f22e7e9 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/ae073b22acf54a4295a974f91f22e7e9 2024-11-19T17:16:06,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/65957afacc6f41a0aea7489a74d4e1c7 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/65957afacc6f41a0aea7489a74d4e1c7 2024-11-19T17:16:06,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/fcf3959291bc4090a452e333b89f5aab to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/fcf3959291bc4090a452e333b89f5aab 2024-11-19T17:16:06,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/032013f72d0d41ab94abef6ff5244390 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/032013f72d0d41ab94abef6ff5244390 2024-11-19T17:16:06,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/933281e693c64ecf9209f25cc1721947, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/9c1c5a5233ee49cf8b41a84798dc44d2, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/0bfb4432c0d14e7299cc9c75f9483d85, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/63f68dc15f554741bd9751ec5a976de1, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/808f107e297741f695d2b68efcdc50de, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/800c5a6a025241e7955f9286cf83d4b3, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/3d00664c7dfa49ce9357c7910ce42fa4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/a1bba2a1ff4d4d929f5f5dc9e08497a8, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/d1114a6dcce44287899832b7e2e94931, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/bba8d888176f4dd9acd198bab662e207, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/72a57aa8e0b542d3b76e696d5fc6601b, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/1e1e225701094a9980da093f7914c21c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/a1600446b9ea4a58b8f64869825e3fe7, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/16545560aea04d73a6c1a8b542310063, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/de8a64b17e244f46a8c9e4c113e1ac06, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/1de74ab3e9734d80815eced24a576aaf, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/50a27562768b4a389325a10fd96c5d85, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/273eaf40f68044b6bfb346dd652711e2, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/5550ce08efaa4caa8e21f655dcd2b6c4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/68c26cc6f4b64ace9e423a599a230d3e, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/59c6a4bac5e045558a98308a772d4582, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/d3468040c2ea428e8e77f36845664096, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/9e0f5b4f9f304a728ab901136ae72db2, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/3a4b2f2deb8f446aa821b2a6ad53c1f3, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/bebfe8392a7545eda8da691c3bd9c100, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/aa3efa7cbca44a04afb73307485a19c4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/10cd31bc4eff4d5b9592dd146ee10a56, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/16d5e941dbd1496daeb5bca2c81a553b] to archive 2024-11-19T17:16:06,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:16:06,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/933281e693c64ecf9209f25cc1721947 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/933281e693c64ecf9209f25cc1721947 2024-11-19T17:16:06,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/9c1c5a5233ee49cf8b41a84798dc44d2 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/9c1c5a5233ee49cf8b41a84798dc44d2 2024-11-19T17:16:06,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/0bfb4432c0d14e7299cc9c75f9483d85 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/0bfb4432c0d14e7299cc9c75f9483d85 2024-11-19T17:16:06,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/63f68dc15f554741bd9751ec5a976de1 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/63f68dc15f554741bd9751ec5a976de1 2024-11-19T17:16:06,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/808f107e297741f695d2b68efcdc50de to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/808f107e297741f695d2b68efcdc50de 2024-11-19T17:16:06,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/800c5a6a025241e7955f9286cf83d4b3 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/800c5a6a025241e7955f9286cf83d4b3 2024-11-19T17:16:06,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/3d00664c7dfa49ce9357c7910ce42fa4 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/3d00664c7dfa49ce9357c7910ce42fa4 2024-11-19T17:16:06,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/a1bba2a1ff4d4d929f5f5dc9e08497a8 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/a1bba2a1ff4d4d929f5f5dc9e08497a8 2024-11-19T17:16:06,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/d1114a6dcce44287899832b7e2e94931 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/d1114a6dcce44287899832b7e2e94931 2024-11-19T17:16:06,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/bba8d888176f4dd9acd198bab662e207 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/bba8d888176f4dd9acd198bab662e207 2024-11-19T17:16:06,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/72a57aa8e0b542d3b76e696d5fc6601b to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/72a57aa8e0b542d3b76e696d5fc6601b 2024-11-19T17:16:06,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/1e1e225701094a9980da093f7914c21c to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/1e1e225701094a9980da093f7914c21c 2024-11-19T17:16:06,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/a1600446b9ea4a58b8f64869825e3fe7 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/a1600446b9ea4a58b8f64869825e3fe7 2024-11-19T17:16:06,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/16545560aea04d73a6c1a8b542310063 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/16545560aea04d73a6c1a8b542310063 2024-11-19T17:16:06,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/de8a64b17e244f46a8c9e4c113e1ac06 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/de8a64b17e244f46a8c9e4c113e1ac06 2024-11-19T17:16:06,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/1de74ab3e9734d80815eced24a576aaf to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/1de74ab3e9734d80815eced24a576aaf 2024-11-19T17:16:06,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/50a27562768b4a389325a10fd96c5d85 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/50a27562768b4a389325a10fd96c5d85 2024-11-19T17:16:06,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/273eaf40f68044b6bfb346dd652711e2 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/273eaf40f68044b6bfb346dd652711e2 2024-11-19T17:16:06,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/5550ce08efaa4caa8e21f655dcd2b6c4 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/5550ce08efaa4caa8e21f655dcd2b6c4 2024-11-19T17:16:06,516 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/68c26cc6f4b64ace9e423a599a230d3e to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/68c26cc6f4b64ace9e423a599a230d3e 2024-11-19T17:16:06,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/59c6a4bac5e045558a98308a772d4582 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/59c6a4bac5e045558a98308a772d4582 2024-11-19T17:16:06,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/d3468040c2ea428e8e77f36845664096 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/d3468040c2ea428e8e77f36845664096 2024-11-19T17:16:06,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/9e0f5b4f9f304a728ab901136ae72db2 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/9e0f5b4f9f304a728ab901136ae72db2 2024-11-19T17:16:06,519 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/3a4b2f2deb8f446aa821b2a6ad53c1f3 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/3a4b2f2deb8f446aa821b2a6ad53c1f3 2024-11-19T17:16:06,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/bebfe8392a7545eda8da691c3bd9c100 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/bebfe8392a7545eda8da691c3bd9c100 2024-11-19T17:16:06,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/aa3efa7cbca44a04afb73307485a19c4 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/aa3efa7cbca44a04afb73307485a19c4 2024-11-19T17:16:06,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/10cd31bc4eff4d5b9592dd146ee10a56 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/10cd31bc4eff4d5b9592dd146ee10a56 2024-11-19T17:16:06,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/16d5e941dbd1496daeb5bca2c81a553b to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/16d5e941dbd1496daeb5bca2c81a553b 2024-11-19T17:16:06,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/bc4f2baf30734bc4a4b7d28849d5f68f, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/e08d583706fb44d1829d547ab901c005, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/956c1dc238604c03bca6f5a067da8aad, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/287798b5c69f4bf99c3dc9418c9e2bfd, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/32ef8153b7734b5fa808aad16fe230c3, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/39fa028e6b6c42c5988717438aff339e, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/adf4cd39e7b0475c9495d9afe2b76fa4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/577fd51d3e8044b0b1ed0fd09e0ead7f, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/f3772feaa1c143b4b57cf21a72e419fd, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/3d0fad72e2084308830ec4b787979173, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/05738fe53bf449f194c6d5b80e4a2554, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/c8eea07bdc6d421fa3a014217ec76d3d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/d930934bca5f442e83a90d87283bc23d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/91f7582d059240a49d7c15fa9a54c52d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/a0b7dcac19b643adaa80e00faf629b31, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/95b9df38a0024d3a8de3c940d0d8481a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/83eab07641494e8e8352a86f8d0d2f98, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/c2adcb7775ee49e2a4403947dbec6948, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/e6d3531317954c37a74cea6e91ea0de9, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/7e21052ce3c64cd985c2765326bfaff4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/af254ce784584828b33832d5c44cd801, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/cb36d69ab8eb46c8849cf0540710388c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/b91668d8801c4b33847d27828436b3a2, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/5d3c8d4378894f2e95372115b3eb54a6, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/e152bb719ab848faa599155301de30a9, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/cf4000da4abf4850b4df9012c9df3d12, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/9ce66764de38421da63dbd34d84e6b5d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/0f236016d7b745e9b0a44b5bfbf91e5b] to archive 2024-11-19T17:16:06,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:16:06,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/bc4f2baf30734bc4a4b7d28849d5f68f to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/bc4f2baf30734bc4a4b7d28849d5f68f 2024-11-19T17:16:06,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/e08d583706fb44d1829d547ab901c005 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/e08d583706fb44d1829d547ab901c005 2024-11-19T17:16:06,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/956c1dc238604c03bca6f5a067da8aad to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/956c1dc238604c03bca6f5a067da8aad 2024-11-19T17:16:06,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/287798b5c69f4bf99c3dc9418c9e2bfd to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/287798b5c69f4bf99c3dc9418c9e2bfd 2024-11-19T17:16:06,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/32ef8153b7734b5fa808aad16fe230c3 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/32ef8153b7734b5fa808aad16fe230c3 2024-11-19T17:16:06,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/39fa028e6b6c42c5988717438aff339e to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/39fa028e6b6c42c5988717438aff339e 2024-11-19T17:16:06,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/adf4cd39e7b0475c9495d9afe2b76fa4 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/adf4cd39e7b0475c9495d9afe2b76fa4 2024-11-19T17:16:06,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/577fd51d3e8044b0b1ed0fd09e0ead7f to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/577fd51d3e8044b0b1ed0fd09e0ead7f 2024-11-19T17:16:06,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/f3772feaa1c143b4b57cf21a72e419fd to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/f3772feaa1c143b4b57cf21a72e419fd 2024-11-19T17:16:06,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/3d0fad72e2084308830ec4b787979173 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/3d0fad72e2084308830ec4b787979173 2024-11-19T17:16:06,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/05738fe53bf449f194c6d5b80e4a2554 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/05738fe53bf449f194c6d5b80e4a2554 2024-11-19T17:16:06,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/c8eea07bdc6d421fa3a014217ec76d3d to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/c8eea07bdc6d421fa3a014217ec76d3d 2024-11-19T17:16:06,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/d930934bca5f442e83a90d87283bc23d to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/d930934bca5f442e83a90d87283bc23d 2024-11-19T17:16:06,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/91f7582d059240a49d7c15fa9a54c52d to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/91f7582d059240a49d7c15fa9a54c52d 2024-11-19T17:16:06,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/a0b7dcac19b643adaa80e00faf629b31 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/a0b7dcac19b643adaa80e00faf629b31 2024-11-19T17:16:06,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/95b9df38a0024d3a8de3c940d0d8481a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/95b9df38a0024d3a8de3c940d0d8481a 2024-11-19T17:16:06,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/83eab07641494e8e8352a86f8d0d2f98 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/83eab07641494e8e8352a86f8d0d2f98 2024-11-19T17:16:06,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/c2adcb7775ee49e2a4403947dbec6948 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/c2adcb7775ee49e2a4403947dbec6948 2024-11-19T17:16:06,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/e6d3531317954c37a74cea6e91ea0de9 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/e6d3531317954c37a74cea6e91ea0de9 2024-11-19T17:16:06,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/7e21052ce3c64cd985c2765326bfaff4 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/7e21052ce3c64cd985c2765326bfaff4 2024-11-19T17:16:06,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/af254ce784584828b33832d5c44cd801 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/af254ce784584828b33832d5c44cd801 2024-11-19T17:16:06,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/cb36d69ab8eb46c8849cf0540710388c to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/cb36d69ab8eb46c8849cf0540710388c 2024-11-19T17:16:06,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/b91668d8801c4b33847d27828436b3a2 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/b91668d8801c4b33847d27828436b3a2 2024-11-19T17:16:06,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/5d3c8d4378894f2e95372115b3eb54a6 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/5d3c8d4378894f2e95372115b3eb54a6 2024-11-19T17:16:06,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/e152bb719ab848faa599155301de30a9 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/e152bb719ab848faa599155301de30a9 2024-11-19T17:16:06,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/cf4000da4abf4850b4df9012c9df3d12 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/cf4000da4abf4850b4df9012c9df3d12 2024-11-19T17:16:06,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/9ce66764de38421da63dbd34d84e6b5d to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/9ce66764de38421da63dbd34d84e6b5d 2024-11-19T17:16:06,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/0f236016d7b745e9b0a44b5bfbf91e5b to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/0f236016d7b745e9b0a44b5bfbf91e5b 2024-11-19T17:16:06,548 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/recovered.edits/450.seqid, newMaxSeqId=450, maxSeqId=1 2024-11-19T17:16:06,549 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. 2024-11-19T17:16:06,549 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for 2d49416a7b2a16305b1ea9d2c71b4946: Waiting for close lock at 1732036564906Running coprocessor pre-close hooks at 1732036564906Disabling compacts and flushes for region at 1732036564906Disabling writes for close at 1732036565241 (+335 ms)Obtaining lock to block concurrent updates at 1732036565241Preparing flush snapshotting stores in 2d49416a7b2a16305b1ea9d2c71b4946 at 1732036565241Finished memstore snapshotting TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1732036565242 (+1 ms)Flushing stores of TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946. at 1732036565242Flushing 2d49416a7b2a16305b1ea9d2c71b4946/A: creating writer at 1732036565242Flushing 2d49416a7b2a16305b1ea9d2c71b4946/A: appending metadata at 1732036565244 (+2 ms)Flushing 2d49416a7b2a16305b1ea9d2c71b4946/A: closing flushed file at 1732036565244Flushing 2d49416a7b2a16305b1ea9d2c71b4946/B: creating writer at 1732036565650 (+406 ms)Flushing 2d49416a7b2a16305b1ea9d2c71b4946/B: appending metadata at 1732036565652 (+2 ms)Flushing 2d49416a7b2a16305b1ea9d2c71b4946/B: closing flushed file at 1732036565652Flushing 2d49416a7b2a16305b1ea9d2c71b4946/C: creating writer at 1732036566059 (+407 ms)Flushing 2d49416a7b2a16305b1ea9d2c71b4946/C: appending metadata at 1732036566061 (+2 ms)Flushing 2d49416a7b2a16305b1ea9d2c71b4946/C: closing flushed file at 1732036566061Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73bec223: reopening flushed file at 1732036566467 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f592c56: reopening flushed file at 1732036566470 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@677da97b: reopening flushed file at 1732036566473 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 2d49416a7b2a16305b1ea9d2c71b4946 in 1235ms, sequenceid=447, compaction requested=true at 1732036566476 (+3 ms)Writing region close event to WAL at 1732036566546 (+70 ms)Running coprocessor post-close hooks at 1732036566549 (+3 ms)Closed at 1732036566549 2024-11-19T17:16:06,550 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed 2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:16:06,551 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=2d49416a7b2a16305b1ea9d2c71b4946, regionState=CLOSED 2024-11-19T17:16:06,552 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 because future has completed 2024-11-19T17:16:06,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-19T17:16:06,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure 2d49416a7b2a16305b1ea9d2c71b4946, server=280a42d09ec8,33873,1732036422473 in 1.7980 sec 2024-11-19T17:16:06,555 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-19T17:16:06,555 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d49416a7b2a16305b1ea9d2c71b4946, UNASSIGN in 1.8030 sec 2024-11-19T17:16:06,556 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-19T17:16:06,556 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8050 sec 2024-11-19T17:16:06,557 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732036566557"}]},"ts":"1732036566557"} 2024-11-19T17:16:06,558 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T17:16:06,558 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T17:16:06,560 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8130 sec 2024-11-19T17:16:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-19T17:16:06,904 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:16:06,904 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T17:16:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:16:06,906 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=151, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:16:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-19T17:16:06,906 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=151, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:16:06,909 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:16:06,910 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A, FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B, FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C, FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/recovered.edits] 2024-11-19T17:16:06,912 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/0085815c2edd4a1ba8b560e22d374af4 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/0085815c2edd4a1ba8b560e22d374af4 2024-11-19T17:16:06,913 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/69de6df2dfa14c2791cff560e4d1db4c to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/69de6df2dfa14c2791cff560e4d1db4c 2024-11-19T17:16:06,914 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/776dc0eb845e4b6992255abe6b90d41a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/776dc0eb845e4b6992255abe6b90d41a 2024-11-19T17:16:06,914 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/ad87416562dc4db3bcfc204fd5c45022 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/A/ad87416562dc4db3bcfc204fd5c45022 2024-11-19T17:16:06,916 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/3179271f756347a4bf923fd94f0a0104 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/3179271f756347a4bf923fd94f0a0104 2024-11-19T17:16:06,917 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/6ff45f5ce3344ccdb609c30c73b6d787 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/6ff45f5ce3344ccdb609c30c73b6d787 2024-11-19T17:16:06,918 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/bdbb254438034753a1d0ec80a4305177 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/bdbb254438034753a1d0ec80a4305177 2024-11-19T17:16:06,918 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/f40318fb38f34d4eb4b5dcbe75a079cc to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/B/f40318fb38f34d4eb4b5dcbe75a079cc 2024-11-19T17:16:06,920 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/236c2b3826b54e9aaffcaec001b16a0f to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/236c2b3826b54e9aaffcaec001b16a0f 2024-11-19T17:16:06,921 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/7843760307be49d29db25c279e8cb6f6 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/7843760307be49d29db25c279e8cb6f6 2024-11-19T17:16:06,922 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/9d4073978651455894cbb8dfd03cd45d to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/9d4073978651455894cbb8dfd03cd45d 2024-11-19T17:16:06,922 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/dcd2fbbd92344b8cb01007d8b35ed35e to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/C/dcd2fbbd92344b8cb01007d8b35ed35e 2024-11-19T17:16:06,924 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/recovered.edits/450.seqid to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946/recovered.edits/450.seqid 2024-11-19T17:16:06,925 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/2d49416a7b2a16305b1ea9d2c71b4946 2024-11-19T17:16:06,925 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T17:16:06,926 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=151, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:16:06,928 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T17:16:06,929 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T17:16:06,930 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=151, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:16:06,930 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T17:16:06,930 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732036566930"}]},"ts":"9223372036854775807"} 2024-11-19T17:16:06,932 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T17:16:06,932 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2d49416a7b2a16305b1ea9d2c71b4946, NAME => 'TestAcidGuarantees,,1732036541573.2d49416a7b2a16305b1ea9d2c71b4946.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T17:16:06,932 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T17:16:06,932 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732036566932"}]},"ts":"9223372036854775807"} 2024-11-19T17:16:06,933 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T17:16:06,934 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=151, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:16:06,935 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 30 msec 2024-11-19T17:16:07,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-19T17:16:07,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T17:16:07,014 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:16:07,023 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 251), OpenFileDescriptor=449 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=366 (was 380), ProcessCount=11 (was 11), AvailableMemoryMB=6096 (was 6160) 2024-11-19T17:16:07,031 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=366, ProcessCount=11, AvailableMemoryMB=6096 2024-11-19T17:16:07,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T17:16:07,033 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T17:16:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T17:16:07,035 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T17:16:07,035 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:07,035 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 152 2024-11-19T17:16:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T17:16:07,036 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T17:16:07,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742416_1592 (size=1061) 2024-11-19T17:16:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T17:16:07,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T17:16:07,442 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e 2024-11-19T17:16:07,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742417_1593 (size=53) 2024-11-19T17:16:07,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T17:16:07,846 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:16:07,847 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a6bc673cb3ab3290d1b783693a45a9bb, disabling compactions & flushes 2024-11-19T17:16:07,847 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:07,847 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:07,847 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. after waiting 0 ms 2024-11-19T17:16:07,847 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:07,847 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:07,847 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a6bc673cb3ab3290d1b783693a45a9bb: Waiting for close lock at 1732036567847Disabling compacts and flushes for region at 1732036567847Disabling writes for close at 1732036567847Writing region close event to WAL at 1732036567847Closed at 1732036567847 2024-11-19T17:16:07,848 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T17:16:07,848 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732036567848"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732036567848"}]},"ts":"1732036567848"} 2024-11-19T17:16:07,849 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T17:16:07,850 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T17:16:07,850 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732036567850"}]},"ts":"1732036567850"} 2024-11-19T17:16:07,851 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T17:16:07,851 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6bc673cb3ab3290d1b783693a45a9bb, ASSIGN}] 2024-11-19T17:16:07,852 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6bc673cb3ab3290d1b783693a45a9bb, ASSIGN 2024-11-19T17:16:07,853 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6bc673cb3ab3290d1b783693a45a9bb, ASSIGN; state=OFFLINE, location=280a42d09ec8,33873,1732036422473; forceNewPlan=false, retain=false 2024-11-19T17:16:08,003 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=a6bc673cb3ab3290d1b783693a45a9bb, regionState=OPENING, regionLocation=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:08,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6bc673cb3ab3290d1b783693a45a9bb, ASSIGN because future has completed 2024-11-19T17:16:08,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473}] 2024-11-19T17:16:08,158 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:08,159 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7752): Opening region: {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:16:08,159 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:08,159 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:16:08,159 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7794): checking encryption for a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:08,159 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7797): checking classloading for a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:08,160 INFO [StoreOpener-a6bc673cb3ab3290d1b783693a45a9bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:08,161 INFO [StoreOpener-a6bc673cb3ab3290d1b783693a45a9bb-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T17:16:08,161 INFO [StoreOpener-a6bc673cb3ab3290d1b783693a45a9bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6bc673cb3ab3290d1b783693a45a9bb columnFamilyName A 2024-11-19T17:16:08,161 DEBUG [StoreOpener-a6bc673cb3ab3290d1b783693a45a9bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:08,162 INFO [StoreOpener-a6bc673cb3ab3290d1b783693a45a9bb-1 {}] regionserver.HStore(327): Store=a6bc673cb3ab3290d1b783693a45a9bb/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:16:08,162 INFO [StoreOpener-a6bc673cb3ab3290d1b783693a45a9bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:08,162 INFO [StoreOpener-a6bc673cb3ab3290d1b783693a45a9bb-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T17:16:08,162 INFO [StoreOpener-a6bc673cb3ab3290d1b783693a45a9bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6bc673cb3ab3290d1b783693a45a9bb columnFamilyName B 2024-11-19T17:16:08,163 DEBUG [StoreOpener-a6bc673cb3ab3290d1b783693a45a9bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:08,163 INFO [StoreOpener-a6bc673cb3ab3290d1b783693a45a9bb-1 {}] regionserver.HStore(327): Store=a6bc673cb3ab3290d1b783693a45a9bb/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:16:08,163 INFO [StoreOpener-a6bc673cb3ab3290d1b783693a45a9bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:08,164 INFO [StoreOpener-a6bc673cb3ab3290d1b783693a45a9bb-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T17:16:08,164 INFO [StoreOpener-a6bc673cb3ab3290d1b783693a45a9bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6bc673cb3ab3290d1b783693a45a9bb columnFamilyName C 2024-11-19T17:16:08,164 DEBUG [StoreOpener-a6bc673cb3ab3290d1b783693a45a9bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:08,164 INFO [StoreOpener-a6bc673cb3ab3290d1b783693a45a9bb-1 {}] regionserver.HStore(327): Store=a6bc673cb3ab3290d1b783693a45a9bb/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:16:08,164 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:08,164 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1038): replaying wal for a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:08,165 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:08,165 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:08,166 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1048): stopping wal replay for a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:08,166 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1060): Cleaning up temporary data for a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:08,166 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T17:16:08,167 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1093): writing seq id for a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:08,168 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T17:16:08,169 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1114): Opened a6bc673cb3ab3290d1b783693a45a9bb; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64665922, jitterRate=-0.03640267252922058}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T17:16:08,169 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:08,169 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1006): Region open journal for a6bc673cb3ab3290d1b783693a45a9bb: Running coprocessor pre-open hook at 1732036568159Writing region info on filesystem at 1732036568159Initializing all the Stores at 1732036568160 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036568160Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036568160Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036568160Cleaning up temporary data from old regions at 1732036568166 (+6 ms)Running coprocessor post-open hooks at 1732036568169 (+3 ms)Region opened successfully at 1732036568169 2024-11-19T17:16:08,170 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., pid=154, masterSystemTime=1732036568156 2024-11-19T17:16:08,171 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:08,171 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:08,172 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=a6bc673cb3ab3290d1b783693a45a9bb, regionState=OPEN, openSeqNum=2, regionLocation=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:08,173 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=154, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 because future has completed 2024-11-19T17:16:08,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T17:16:08,175 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-19T17:16:08,175 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; OpenRegionProcedure a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 in 168 msec 2024-11-19T17:16:08,176 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-19T17:16:08,176 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6bc673cb3ab3290d1b783693a45a9bb, ASSIGN in 324 msec 2024-11-19T17:16:08,177 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T17:16:08,177 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732036568177"}]},"ts":"1732036568177"} 2024-11-19T17:16:08,178 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T17:16:08,178 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T17:16:08,179 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1450 sec 2024-11-19T17:16:09,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T17:16:09,184 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:16:09,185 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e2cba0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:09,185 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:16:09,185 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:16:09,186 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:16:09,186 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:16:09,186 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:16:09,186 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5843b284, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:09,187 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:16:09,187 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:16:09,187 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:09,188 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:16:09,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45bae53a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:09,188 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:16:09,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:16:09,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:16:09,190 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:16:09,191 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T17:16:09,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 280a42d09ec8,40603,1732036421715 2024-11-19T17:16:09,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5d606323 2024-11-19T17:16:09,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T17:16:09,192 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T17:16:09,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T17:16:09,193 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T17:16:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=155, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T17:16:09,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742418_1594 (size=1097) 2024-11-19T17:16:09,603 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-19T17:16:09,603 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-19T17:16:09,605 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T17:16:09,607 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6bc673cb3ab3290d1b783693a45a9bb, REOPEN/MOVE}] 2024-11-19T17:16:09,608 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6bc673cb3ab3290d1b783693a45a9bb, REOPEN/MOVE 2024-11-19T17:16:09,608 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=a6bc673cb3ab3290d1b783693a45a9bb, regionState=CLOSING, regionLocation=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:09,609 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6bc673cb3ab3290d1b783693a45a9bb, REOPEN/MOVE because future has completed 2024-11-19T17:16:09,609 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T17:16:09,609 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE, hasLock=false; CloseRegionProcedure a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473}] 2024-11-19T17:16:09,761 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(122): Close a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:09,761 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T17:16:09,761 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1722): Closing a6bc673cb3ab3290d1b783693a45a9bb, disabling compactions & flushes 2024-11-19T17:16:09,761 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:09,762 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:09,762 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. after waiting 0 ms 2024-11-19T17:16:09,762 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:09,765 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T17:16:09,765 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:09,765 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1676): Region close journal for a6bc673cb3ab3290d1b783693a45a9bb: Waiting for close lock at 1732036569761Running coprocessor pre-close hooks at 1732036569761Disabling compacts and flushes for region at 1732036569761Disabling writes for close at 1732036569762 (+1 ms)Writing region close event to WAL at 1732036569762Running coprocessor post-close hooks at 1732036569765 (+3 ms)Closed at 1732036569765 2024-11-19T17:16:09,765 WARN [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegionServer(3299): Not adding moved region record: a6bc673cb3ab3290d1b783693a45a9bb to self. 2024-11-19T17:16:09,767 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(157): Closed a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:09,767 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=a6bc673cb3ab3290d1b783693a45a9bb, regionState=CLOSED 2024-11-19T17:16:09,768 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=158, ppid=157, state=RUNNABLE, hasLock=false; CloseRegionProcedure a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 because future has completed 2024-11-19T17:16:09,770 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=158, resume processing ppid=157 2024-11-19T17:16:09,770 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, ppid=157, state=SUCCESS, hasLock=false; CloseRegionProcedure a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 in 160 msec 2024-11-19T17:16:09,771 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6bc673cb3ab3290d1b783693a45a9bb, REOPEN/MOVE; state=CLOSED, location=280a42d09ec8,33873,1732036422473; forceNewPlan=false, retain=true 2024-11-19T17:16:09,921 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=a6bc673cb3ab3290d1b783693a45a9bb, regionState=OPENING, regionLocation=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:09,923 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6bc673cb3ab3290d1b783693a45a9bb, REOPEN/MOVE because future has completed 2024-11-19T17:16:09,923 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=157, state=RUNNABLE, hasLock=false; OpenRegionProcedure a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473}] 2024-11-19T17:16:10,078 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:10,078 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(7752): Opening region: {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:16:10,079 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:10,079 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:16:10,079 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(7794): checking encryption for a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:10,079 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(7797): checking classloading for a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:10,080 INFO [StoreOpener-a6bc673cb3ab3290d1b783693a45a9bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:10,080 INFO [StoreOpener-a6bc673cb3ab3290d1b783693a45a9bb-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T17:16:10,081 INFO [StoreOpener-a6bc673cb3ab3290d1b783693a45a9bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6bc673cb3ab3290d1b783693a45a9bb columnFamilyName A 2024-11-19T17:16:10,082 DEBUG [StoreOpener-a6bc673cb3ab3290d1b783693a45a9bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:10,082 INFO [StoreOpener-a6bc673cb3ab3290d1b783693a45a9bb-1 {}] regionserver.HStore(327): Store=a6bc673cb3ab3290d1b783693a45a9bb/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:16:10,083 INFO [StoreOpener-a6bc673cb3ab3290d1b783693a45a9bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:10,083 INFO [StoreOpener-a6bc673cb3ab3290d1b783693a45a9bb-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T17:16:10,083 INFO [StoreOpener-a6bc673cb3ab3290d1b783693a45a9bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6bc673cb3ab3290d1b783693a45a9bb columnFamilyName B 2024-11-19T17:16:10,083 DEBUG [StoreOpener-a6bc673cb3ab3290d1b783693a45a9bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:10,084 INFO [StoreOpener-a6bc673cb3ab3290d1b783693a45a9bb-1 {}] regionserver.HStore(327): Store=a6bc673cb3ab3290d1b783693a45a9bb/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:16:10,084 INFO [StoreOpener-a6bc673cb3ab3290d1b783693a45a9bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:10,084 INFO [StoreOpener-a6bc673cb3ab3290d1b783693a45a9bb-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T17:16:10,085 INFO [StoreOpener-a6bc673cb3ab3290d1b783693a45a9bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6bc673cb3ab3290d1b783693a45a9bb columnFamilyName C 2024-11-19T17:16:10,085 DEBUG [StoreOpener-a6bc673cb3ab3290d1b783693a45a9bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:10,085 INFO [StoreOpener-a6bc673cb3ab3290d1b783693a45a9bb-1 {}] regionserver.HStore(327): Store=a6bc673cb3ab3290d1b783693a45a9bb/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:16:10,085 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:10,085 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1038): replaying wal for a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:10,086 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:10,086 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:10,087 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1048): stopping wal replay for a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:10,087 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1060): Cleaning up temporary data for a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:10,088 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T17:16:10,089 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1093): writing seq id for a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:10,090 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1114): Opened a6bc673cb3ab3290d1b783693a45a9bb; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74790058, jitterRate=0.11445870995521545}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T17:16:10,090 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:10,090 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1006): Region open journal for a6bc673cb3ab3290d1b783693a45a9bb: Running coprocessor pre-open hook at 1732036570079Writing region info on filesystem at 1732036570079Initializing all the Stores at 1732036570079Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036570079Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036570080 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732036570080Cleaning up temporary data from old regions at 1732036570087 (+7 ms)Running coprocessor post-open hooks at 1732036570090 (+3 ms)Region opened successfully at 1732036570090 2024-11-19T17:16:10,091 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., pid=159, masterSystemTime=1732036570075 2024-11-19T17:16:10,093 DEBUG [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:10,093 INFO [RS_OPEN_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:10,094 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=a6bc673cb3ab3290d1b783693a45a9bb, regionState=OPEN, openSeqNum=5, regionLocation=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:10,095 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=159, ppid=157, state=RUNNABLE, hasLock=false; OpenRegionProcedure a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 because future has completed 2024-11-19T17:16:10,098 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=157 2024-11-19T17:16:10,098 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=157, state=SUCCESS, hasLock=false; OpenRegionProcedure a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 in 173 msec 2024-11-19T17:16:10,099 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-19T17:16:10,099 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6bc673cb3ab3290d1b783693a45a9bb, REOPEN/MOVE in 491 msec 2024-11-19T17:16:10,101 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=156, resume processing ppid=155 2024-11-19T17:16:10,101 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, ppid=155, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 495 msec 2024-11-19T17:16:10,103 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 908 msec 2024-11-19T17:16:10,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=155 2024-11-19T17:16:10,103 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:16:10,104 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@744b0f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:10,104 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:16:10,104 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:16:10,105 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:16:10,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:16:10,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:16:10,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7555afdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:10,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:16:10,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:16:10,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:10,106 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35424, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:16:10,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74df12f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:10,107 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143fb08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:10,107 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:16:10,107 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:16:10,108 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:16:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:16:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:16:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3060d413, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:16:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:16:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:10,109 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:16:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bff3e1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:10,110 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:10,110 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:16:10,110 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:16:10,111 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:16:10,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:16:10,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:16:10,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@100c7bb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:10,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:16:10,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:16:10,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:10,112 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:16:10,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ff0d234, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:10,112 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f72e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:10,113 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:16:10,113 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:16:10,113 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:16:10,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:16:10,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:16:10,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e5e7dcd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:10,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:16:10,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:16:10,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:10,114 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:16:10,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a9a6f74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:10,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47173f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:10,115 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:16:10,115 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:16:10,116 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:16:10,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:16:10,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:16:10,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c4aa72b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:10,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:16:10,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:16:10,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:10,117 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:16:10,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@783c4561, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:10,118 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7122db2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:10,118 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:16:10,118 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:16:10,118 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:16:10,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:16:10,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:16:10,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3127dfaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:10,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:16:10,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:16:10,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:10,119 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:16:10,120 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@638864cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:10,120 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:10,120 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:16:10,120 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:16:10,121 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:16:10,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:16:10,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:16:10,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c2dd26c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:10,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:16:10,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:16:10,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:10,122 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:16:10,122 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f67f312, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:10,123 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:10,123 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:16:10,123 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:16:10,123 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:16:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:16:10,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:16:10,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eb5f7e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:10,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:16:10,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:16:10,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:10,125 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:16:10,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73604b25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:10,126 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d0e9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:10,126 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:16:10,126 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:16:10,126 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:16:10,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:16:10,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:16:10,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36668b36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:10,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:16:10,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:16:10,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:10,127 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:16:10,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f21e915, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:10,128 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9a2803, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:10,128 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 280a42d09ec8,40603,-1 for getting cluster id 2024-11-19T17:16:10,128 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:16:10,129 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e13b793f-957c-4380-9440-a4e203c4ee51' 2024-11-19T17:16:10,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:16:10,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e13b793f-957c-4380-9440-a4e203c4ee51" 2024-11-19T17:16:10,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a384b1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:10,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [280a42d09ec8,40603,-1] 2024-11-19T17:16:10,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:16:10,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:10,130 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:16:10,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b0ec73c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:16:10,131 DEBUG [Thread-2637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:16:10,131 DEBUG [Thread-2620 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:16:10,131 DEBUG [Thread-2624 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:16:10,132 DEBUG [Thread-2628 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:16:10,132 DEBUG [Thread-2635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:16:10,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:16:10,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:16:10,132 DEBUG [Thread-2639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:16:10,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:16:10,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:16:10,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:16:10,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:16:10,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:16:10,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:16:10,134 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:16:10,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:16:10,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:16:10,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:16:10,134 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:16:10,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:16:10,135 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:16:10,135 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:16:10,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5] 2024-11-19T17:16:10,137 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:16:10,137 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:16:10,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5] 2024-11-19T17:16:10,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5] 2024-11-19T17:16:10,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5] 2024-11-19T17:16:10,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5] 2024-11-19T17:16:10,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5] 2024-11-19T17:16:10,144 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:16:10,144 DEBUG [Thread-2631 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:16:10,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-19T17:16:10,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:16:10,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:10,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:16:10,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6bc673cb3ab3290d1b783693a45a9bb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T17:16:10,145 DEBUG [Thread-2626 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:16:10,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=A 2024-11-19T17:16:10,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:10,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=B 2024-11-19T17:16:10,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:10,146 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:16:10,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=C 2024-11-19T17:16:10,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:10,147 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:16:10,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:16:10,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T17:16:10,149 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:16:10,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5] 2024-11-19T17:16:10,152 DEBUG [Thread-2626 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:16:10,153 DEBUG [Thread-2626 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:16:10,154 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:16:10,157 DEBUG [Thread-2626 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5] 2024-11-19T17:16:10,165 DEBUG [Thread-2633 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:16:10,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:16:10,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:16:10,167 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:16:10,169 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5] 2024-11-19T17:16:10,172 DEBUG [Thread-2622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:16:10,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=280a42d09ec8,33873,1732036422473, seqNum=-1] 2024-11-19T17:16:10,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:16:10,174 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:16:10,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5] 2024-11-19T17:16:10,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036570160, max timestamp=1732036570185Num uniques 859; 2024-11-19T17:16:10,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:16:10,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036570160, max timestamp=1732036570185Num uniques 859; 2024-11-19T17:16:10,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:16:10,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036570160, max timestamp=1732036570186Num uniques 859; 2024-11-19T17:16:10,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:16:10,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119203e2c03315b4adb9111adfd29b306d1_a6bc673cb3ab3290d1b783693a45a9bb is 50, key is test_row_0/A:col10/1732036570141/Put/seqid=0 2024-11-19T17:16:10,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742419_1595 (size=12154) 2024-11-19T17:16:10,203 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:10,207 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119203e2c03315b4adb9111adfd29b306d1_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119203e2c03315b4adb9111adfd29b306d1_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:10,209 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/7d0505b848ff426dafddf8790c31383f, store: [table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:10,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/7d0505b848ff426dafddf8790c31383f is 175, key is test_row_0/A:col10/1732036570141/Put/seqid=0 2024-11-19T17:16:10,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742420_1596 (size=30955) 2024-11-19T17:16:10,213 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/7d0505b848ff426dafddf8790c31383f 2024-11-19T17:16:10,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:10,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036630208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:10,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036630208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:10,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:10,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036630212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:10,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:10,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34418 deadline: 1732036630214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:10,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:10,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34398 deadline: 1732036630214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:10,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:10,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/f637db8a86af408f97f22de974db9de4 is 50, key is test_row_0/B:col10/1732036570141/Put/seqid=0 2024-11-19T17:16:10,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742421_1597 (size=12001) 2024-11-19T17:16:10,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T17:16:10,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T17:16:10,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:10,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:10,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:10,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:10,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:10,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:10,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036630324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:10,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:10,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036630324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:10,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:10,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:10,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34418 deadline: 1732036630324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:10,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:10,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:10,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:10,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34398 deadline: 1732036630324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:10,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:10,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036630325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:10,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:10,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:10,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T17:16:10,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:10,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:10,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:10,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:10,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:10,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T17:16:10,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:10,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036630533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:10,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:10,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:10,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036630533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:10,538 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:10,538 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,538 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:10,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34418 deadline: 1732036630533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:10,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:10,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:10,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34398 deadline: 1732036630534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:10,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:10,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036630534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:10,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:10,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:10,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T17:16:10,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:10,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:10,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:10,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:10,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:10,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:10,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/f637db8a86af408f97f22de974db9de4 2024-11-19T17:16:10,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/ac84d178d35d4a15a94bae66f890db1d is 50, key is test_row_0/C:col10/1732036570141/Put/seqid=0 2024-11-19T17:16:10,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742422_1598 (size=12001) 2024-11-19T17:16:10,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/ac84d178d35d4a15a94bae66f890db1d 2024-11-19T17:16:10,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/7d0505b848ff426dafddf8790c31383f as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/7d0505b848ff426dafddf8790c31383f 2024-11-19T17:16:10,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/7d0505b848ff426dafddf8790c31383f, entries=150, sequenceid=14, filesize=30.2 K 2024-11-19T17:16:10,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/f637db8a86af408f97f22de974db9de4 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/f637db8a86af408f97f22de974db9de4 2024-11-19T17:16:10,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/f637db8a86af408f97f22de974db9de4, entries=150, sequenceid=14, filesize=11.7 K 2024-11-19T17:16:10,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/ac84d178d35d4a15a94bae66f890db1d as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/ac84d178d35d4a15a94bae66f890db1d 2024-11-19T17:16:10,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/ac84d178d35d4a15a94bae66f890db1d, entries=150, sequenceid=14, filesize=11.7 K 2024-11-19T17:16:10,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for a6bc673cb3ab3290d1b783693a45a9bb in 541ms, sequenceid=14, compaction requested=false 2024-11-19T17:16:10,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:10,709 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T17:16:10,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T17:16:10,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:10,765 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing a6bc673cb3ab3290d1b783693a45a9bb 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-19T17:16:10,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=A 2024-11-19T17:16:10,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:16:10,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=B 2024-11-19T17:16:10,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:16:10,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=C 2024-11-19T17:16:10,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:16:10,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T17:16:10,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119713bd99daa9443209c789f2bbf57a75d_a6bc673cb3ab3290d1b783693a45a9bb is 50, key is test_row_0/A:col10/1732036570206/Put/seqid=0 2024-11-19T17:16:10,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742423_1599 (size=12154) 2024-11-19T17:16:10,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:10,794 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119713bd99daa9443209c789f2bbf57a75d_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119713bd99daa9443209c789f2bbf57a75d_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:10,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/0e0a721fd23c4bbbab6a272bace38421, store: [table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:10,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/0e0a721fd23c4bbbab6a272bace38421 is 175, key is test_row_0/A:col10/1732036570206/Put/seqid=0 2024-11-19T17:16:10,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742424_1600 (size=30955) 2024-11-19T17:16:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:10,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:10,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:10,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:10,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036630849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:10,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34418 deadline: 1732036630850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:10,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:10,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:10,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:10,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34398 deadline: 1732036630850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:10,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:10,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:10,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036630851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:10,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:10,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:10,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036630855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:10,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:10,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:10,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036630963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:10,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34398 deadline: 1732036630964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:10,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34418 deadline: 1732036630964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:10,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:10,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036630964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:10,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:10,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:10,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036630964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:10,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:11,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036631174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:11,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34398 deadline: 1732036631174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:11,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:11,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:11,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036631174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:11,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:11,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:11,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:11,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:11,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34418 deadline: 1732036631174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:11,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:11,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:11,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:11,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:11,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036631174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:11,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:11,179 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:11,179 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:11,179 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:11,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:11,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:11,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:11,200 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/0e0a721fd23c4bbbab6a272bace38421 2024-11-19T17:16:11,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/8a01da8a613c4d84be127d8e68e44f46 is 50, key is test_row_0/B:col10/1732036570206/Put/seqid=0 2024-11-19T17:16:11,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742425_1601 (size=12001) 2024-11-19T17:16:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T17:16:11,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:11,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036631484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:11,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:11,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:11,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:11,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036631484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:11,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:11,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:11,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:11,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:11,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036631484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:11,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34418 deadline: 1732036631485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:11,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34398 deadline: 1732036631484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:11,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:11,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:11,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:11,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:11,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:11,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:11,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:11,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:11,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:11,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:11,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:11,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:11,619 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/8a01da8a613c4d84be127d8e68e44f46 2024-11-19T17:16:11,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/d6e9b245ec364c89b91069eaec402c4a is 50, key is test_row_0/C:col10/1732036570206/Put/seqid=0 2024-11-19T17:16:11,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742426_1602 (size=12001) 2024-11-19T17:16:11,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036631993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:11,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:11,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:11,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34398 deadline: 1732036631993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:11,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:11,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:11,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:11,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036631993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:11,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:11,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:11,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:11,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:12,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:12,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036632004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:12,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:12,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:12,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:12,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:12,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34418 deadline: 1732036632004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:12,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:12,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:12,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:12,036 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/d6e9b245ec364c89b91069eaec402c4a 2024-11-19T17:16:12,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/0e0a721fd23c4bbbab6a272bace38421 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/0e0a721fd23c4bbbab6a272bace38421 2024-11-19T17:16:12,045 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/0e0a721fd23c4bbbab6a272bace38421, entries=150, sequenceid=51, filesize=30.2 K 2024-11-19T17:16:12,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/8a01da8a613c4d84be127d8e68e44f46 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/8a01da8a613c4d84be127d8e68e44f46 2024-11-19T17:16:12,053 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/8a01da8a613c4d84be127d8e68e44f46, entries=150, sequenceid=51, filesize=11.7 K 2024-11-19T17:16:12,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/d6e9b245ec364c89b91069eaec402c4a as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/d6e9b245ec364c89b91069eaec402c4a 2024-11-19T17:16:12,057 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/d6e9b245ec364c89b91069eaec402c4a, entries=150, sequenceid=51, filesize=11.7 K 2024-11-19T17:16:12,057 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for a6bc673cb3ab3290d1b783693a45a9bb in 1292ms, sequenceid=51, compaction requested=false 2024-11-19T17:16:12,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:12,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:12,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-19T17:16:12,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-19T17:16:12,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-19T17:16:12,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9150 sec 2024-11-19T17:16:12,067 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.9210 sec 2024-11-19T17:16:12,072 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T17:16:12,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T17:16:12,294 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:16:12,294 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:16:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-19T17:16:12,297 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:16:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T17:16:12,298 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:16:12,298 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:16:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T17:16:12,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-19T17:16:12,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:12,451 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing a6bc673cb3ab3290d1b783693a45a9bb 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-19T17:16:12,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=A 2024-11-19T17:16:12,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:12,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=B 2024-11-19T17:16:12,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:12,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=C 2024-11-19T17:16:12,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:12,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198cb8c677a0ea4c99b3be6bca8280a983_a6bc673cb3ab3290d1b783693a45a9bb is 50, key is test_row_0/A:col10/1732036570849/Put/seqid=0 2024-11-19T17:16:12,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742427_1603 (size=12154) 2024-11-19T17:16:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T17:16:12,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:12,895 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198cb8c677a0ea4c99b3be6bca8280a983_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198cb8c677a0ea4c99b3be6bca8280a983_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:12,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/d00c358256c84117895aeeaa99ad5cef, store: [table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:12,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/d00c358256c84117895aeeaa99ad5cef is 175, key is test_row_0/A:col10/1732036570849/Put/seqid=0 2024-11-19T17:16:12,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T17:16:12,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742428_1604 (size=30955) 2024-11-19T17:16:13,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:13,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:13,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732036573024, max timestamp=1732036573034Num uniques 859; 2024-11-19T17:16:13,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T17:16:13,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036573026, max timestamp=1732036573034Num uniques 859; 2024-11-19T17:16:13,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:16:13,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036573026, max timestamp=1732036573034Num uniques 859; 2024-11-19T17:16:13,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:16:13,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:13,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036633066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:13,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:13,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:13,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036633068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:13,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:13,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34418 deadline: 1732036633069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:13,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34398 deadline: 1732036633071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:13,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:13,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036633072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:13,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:13,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34418 deadline: 1732036633184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:13,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:13,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036633184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:13,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:13,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:13,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036633184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:13,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:13,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:13,190 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,190 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34398 deadline: 1732036633184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:13,190 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:13,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:13,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036633185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:13,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:13,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:13,341 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/d00c358256c84117895aeeaa99ad5cef 2024-11-19T17:16:13,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/2e9abe86edae40edb805d1a303cb9622 is 50, key is test_row_0/B:col10/1732036570849/Put/seqid=0 2024-11-19T17:16:13,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742429_1605 (size=12001) 2024-11-19T17:16:13,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34418 deadline: 1732036633394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:13,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036633394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036633394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:13,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34398 deadline: 1732036633394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:13,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036633394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:13,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T17:16:13,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:13,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036633704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:13,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:13,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036633704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:13,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:13,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:13,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:13,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34418 deadline: 1732036633704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:13,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34398 deadline: 1732036633704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:13,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:13,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036633713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:13,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:13,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:13,751 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/2e9abe86edae40edb805d1a303cb9622 2024-11-19T17:16:13,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/a506325c528d46aa95bae864101bde45 is 50, key is test_row_0/C:col10/1732036570849/Put/seqid=0 2024-11-19T17:16:13,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742430_1606 (size=12001) 2024-11-19T17:16:14,161 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/a506325c528d46aa95bae864101bde45 2024-11-19T17:16:14,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/d00c358256c84117895aeeaa99ad5cef as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/d00c358256c84117895aeeaa99ad5cef 2024-11-19T17:16:14,167 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/d00c358256c84117895aeeaa99ad5cef, entries=150, sequenceid=60, filesize=30.2 K 2024-11-19T17:16:14,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/2e9abe86edae40edb805d1a303cb9622 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/2e9abe86edae40edb805d1a303cb9622 2024-11-19T17:16:14,171 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/2e9abe86edae40edb805d1a303cb9622, entries=150, sequenceid=60, filesize=11.7 K 2024-11-19T17:16:14,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/a506325c528d46aa95bae864101bde45 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/a506325c528d46aa95bae864101bde45 2024-11-19T17:16:14,174 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/a506325c528d46aa95bae864101bde45, entries=150, sequenceid=60, filesize=11.7 K 2024-11-19T17:16:14,175 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for a6bc673cb3ab3290d1b783693a45a9bb in 1724ms, sequenceid=60, compaction requested=true 2024-11-19T17:16:14,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:14,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:14,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-19T17:16:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-19T17:16:14,178 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-19T17:16:14,178 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8780 sec 2024-11-19T17:16:14,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.8840 sec 2024-11-19T17:16:14,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:14,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6bc673cb3ab3290d1b783693a45a9bb 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-19T17:16:14,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=A 2024-11-19T17:16:14,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T17:16:14,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=B 2024-11-19T17:16:14,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T17:16:14,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=C 2024-11-19T17:16:14,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T17:16:14,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192aa1004e407044ffb73d2f691bfaf0dc_a6bc673cb3ab3290d1b783693a45a9bb is 50, key is test_row_0/A:col10/1732036573067/Put/seqid=0 2024-11-19T17:16:14,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742431_1607 (size=12154) 2024-11-19T17:16:14,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036634224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:14,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:14,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:14,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036634227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:14,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:14,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036634227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:14,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:14,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:14,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34398 deadline: 1732036634228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:14,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:14,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:14,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34418 deadline: 1732036634229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:14,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:14,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:14,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:14,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036634344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:14,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:14,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036634344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:14,347 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,347 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,347 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:14,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:14,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34398 deadline: 1732036634344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:14,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:14,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:14,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34418 deadline: 1732036634344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:14,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T17:16:14,444 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:16:14,445 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:16:14,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-19T17:16:14,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T17:16:14,447 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:16:14,447 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:16:14,447 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:16:14,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T17:16:14,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:14,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036634554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:14,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:14,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:14,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036634554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:14,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:14,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34398 deadline: 1732036634554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:14,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:14,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:14,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34418 deadline: 1732036634554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:14,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:14,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:14,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T17:16:14,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:14,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:14,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:14,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:14,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:14,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:14,630 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:14,633 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192aa1004e407044ffb73d2f691bfaf0dc_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192aa1004e407044ffb73d2f691bfaf0dc_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:14,634 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/2f7287f33446461cbd9ee08cd69180d9, store: [table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:14,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/2f7287f33446461cbd9ee08cd69180d9 is 175, key is test_row_0/A:col10/1732036573067/Put/seqid=0 2024-11-19T17:16:14,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742432_1608 (size=30955) 2024-11-19T17:16:14,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T17:16:14,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:14,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:14,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:14,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:14,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:14,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T17:16:14,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:14,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036634864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:14,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:14,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:14,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036634864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:14,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:14,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34398 deadline: 1732036634864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:14,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:14,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34418 deadline: 1732036634864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:14,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:14,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T17:16:14,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:14,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:14,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:14,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:14,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:14,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:15,039 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/2f7287f33446461cbd9ee08cd69180d9 2024-11-19T17:16:15,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/ccfc5882261a4c7b9e67d9ba9e1300f0 is 50, key is test_row_0/B:col10/1732036573067/Put/seqid=0 2024-11-19T17:16:15,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742433_1609 (size=12001) 2024-11-19T17:16:15,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/ccfc5882261a4c7b9e67d9ba9e1300f0 2024-11-19T17:16:15,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T17:16:15,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:15,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:15,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:15,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:15,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:15,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:15,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/fd389427fe1f40328f013ace7b84111c is 50, key is test_row_0/C:col10/1732036573067/Put/seqid=0 2024-11-19T17:16:15,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742434_1610 (size=12001) 2024-11-19T17:16:15,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/fd389427fe1f40328f013ace7b84111c 2024-11-19T17:16:15,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/2f7287f33446461cbd9ee08cd69180d9 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/2f7287f33446461cbd9ee08cd69180d9 2024-11-19T17:16:15,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/2f7287f33446461cbd9ee08cd69180d9, entries=150, sequenceid=98, filesize=30.2 K 2024-11-19T17:16:15,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T17:16:15,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/ccfc5882261a4c7b9e67d9ba9e1300f0 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/ccfc5882261a4c7b9e67d9ba9e1300f0 2024-11-19T17:16:15,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/ccfc5882261a4c7b9e67d9ba9e1300f0, entries=150, sequenceid=98, filesize=11.7 K 2024-11-19T17:16:15,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/fd389427fe1f40328f013ace7b84111c as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/fd389427fe1f40328f013ace7b84111c 2024-11-19T17:16:15,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/fd389427fe1f40328f013ace7b84111c, entries=150, sequenceid=98, filesize=11.7 K 2024-11-19T17:16:15,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for a6bc673cb3ab3290d1b783693a45a9bb in 866ms, sequenceid=98, compaction requested=true 2024-11-19T17:16:15,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:15,082 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:16:15,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6bc673cb3ab3290d1b783693a45a9bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:16:15,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:15,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,082 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:16:15,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6bc673cb3ab3290d1b783693a45a9bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:16:15,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:15,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6bc673cb3ab3290d1b783693a45a9bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:16:15,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:16:15,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,084 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 123820 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:16:15,084 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): a6bc673cb3ab3290d1b783693a45a9bb/A is initiating minor compaction (all files) 2024-11-19T17:16:15,084 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6bc673cb3ab3290d1b783693a45a9bb/A in TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:15,084 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:16:15,084 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/7d0505b848ff426dafddf8790c31383f, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/0e0a721fd23c4bbbab6a272bace38421, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/d00c358256c84117895aeeaa99ad5cef, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/2f7287f33446461cbd9ee08cd69180d9] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp, totalSize=120.9 K 2024-11-19T17:16:15,084 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:15,084 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): a6bc673cb3ab3290d1b783693a45a9bb/B is initiating minor compaction (all files) 2024-11-19T17:16:15,084 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. files: [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/7d0505b848ff426dafddf8790c31383f, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/0e0a721fd23c4bbbab6a272bace38421, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/d00c358256c84117895aeeaa99ad5cef, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/2f7287f33446461cbd9ee08cd69180d9] 2024-11-19T17:16:15,084 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6bc673cb3ab3290d1b783693a45a9bb/B in TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:15,084 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/f637db8a86af408f97f22de974db9de4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/8a01da8a613c4d84be127d8e68e44f46, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/2e9abe86edae40edb805d1a303cb9622, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/ccfc5882261a4c7b9e67d9ba9e1300f0] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp, totalSize=46.9 K 2024-11-19T17:16:15,084 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d0505b848ff426dafddf8790c31383f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732036570139 2024-11-19T17:16:15,084 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting f637db8a86af408f97f22de974db9de4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732036570139 2024-11-19T17:16:15,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,085 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e0a721fd23c4bbbab6a272bace38421, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732036570206 2024-11-19T17:16:15,085 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a01da8a613c4d84be127d8e68e44f46, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732036570206 2024-11-19T17:16:15,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,085 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting d00c358256c84117895aeeaa99ad5cef, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732036570847 2024-11-19T17:16:15,085 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e9abe86edae40edb805d1a303cb9622, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732036570847 2024-11-19T17:16:15,085 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f7287f33446461cbd9ee08cd69180d9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732036573067 2024-11-19T17:16:15,085 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting ccfc5882261a4c7b9e67d9ba9e1300f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732036573067 2024-11-19T17:16:15,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,091 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:15,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,092 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6bc673cb3ab3290d1b783693a45a9bb#B#compaction#521 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:16:15,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,093 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/4199c00a41ce42dcab467923ee4cd761 is 50, key is test_row_0/B:col10/1732036573067/Put/seqid=0 2024-11-19T17:16:15,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,093 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411191250913e16d64013ab8e841bfd3ca6c4_a6bc673cb3ab3290d1b783693a45a9bb store=[table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:15,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,095 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411191250913e16d64013ab8e841bfd3ca6c4_a6bc673cb3ab3290d1b783693a45a9bb, store=[table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,095 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191250913e16d64013ab8e841bfd3ca6c4_a6bc673cb3ab3290d1b783693a45a9bb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742435_1611 (size=12139) 2024-11-19T17:16:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742436_1612 (size=4469) 2024-11-19T17:16:15,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T17:16:15,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:15,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,211 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing a6bc673cb3ab3290d1b783693a45a9bb 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T17:16:15,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=A 2024-11-19T17:16:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:15,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=B 2024-11-19T17:16:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:15,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=C 2024-11-19T17:16:15,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198abb66ecabf54dc5ba8f41a29b619568_a6bc673cb3ab3290d1b783693a45a9bb is 50, key is test_row_1/A:col10/1732036574227/Put/seqid=0 2024-11-19T17:16:15,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742437_1613 (size=9714) 2024-11-19T17:16:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,226 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198abb66ecabf54dc5ba8f41a29b619568_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198abb66ecabf54dc5ba8f41a29b619568_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/1c6d923ffb1c4ce481f3da7650447edd, store: [table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:15,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/1c6d923ffb1c4ce481f3da7650447edd is 175, key is test_row_1/A:col10/1732036574227/Put/seqid=0 2024-11-19T17:16:15,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742438_1614 (size=22361) 2024-11-19T17:16:15,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,231 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=106, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/1c6d923ffb1c4ce481f3da7650447edd 2024-11-19T17:16:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/8e5eff5b73fa4149a4cd3cb7723db4bc is 50, key is test_row_1/B:col10/1732036574227/Put/seqid=0 2024-11-19T17:16:15,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742439_1615 (size=9657) 2024-11-19T17:16:15,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,242 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/8e5eff5b73fa4149a4cd3cb7723db4bc 2024-11-19T17:16:15,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/e246b35c30eb46b787c366447c0985a8 is 50, key is test_row_1/C:col10/1732036574227/Put/seqid=0 2024-11-19T17:16:15,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742440_1616 (size=9657) 2024-11-19T17:16:15,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:15,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:15,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036575321, max timestamp=1732036575358Num uniques 859; 2024-11-19T17:16:15,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036575321, max timestamp=1732036575358Num uniques 859; 2024-11-19T17:16:15,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:16:15,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:16:15,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036575321, max timestamp=1732036575358Num uniques 859; 2024-11-19T17:16:15,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:16:15,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:15,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34398 deadline: 1732036635395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:15,408 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9968, totalHeapSize=22912, min timestamp=1732036575389, max timestamp=1732036575402Num uniques 1018; 2024-11-19T17:16:15,408 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9968, totalHeapSize=22912, min timestamp=1732036575389, max timestamp=1732036575402Num uniques 1018; 2024-11-19T17:16:15,408 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46608, new segment data size=9968, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T17:16:15,408 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46608, new segment data size=9968, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T17:16:15,408 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9968, totalHeapSize=22912, min timestamp=1732036575389, max timestamp=1732036575402Num uniques 1018; 2024-11-19T17:16:15,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46608, new segment data size=9968, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T17:16:15,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:15,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:15,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036635437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:15,480 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,480 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,480 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:15,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732036575426, max timestamp=1732036575449Num uniques 1077; 2024-11-19T17:16:15,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-19T17:16:15,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732036575426, max timestamp=1732036575449Num uniques 1077; 2024-11-19T17:16:15,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732036575426, max timestamp=1732036575449Num uniques 1077; 2024-11-19T17:16:15,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=6870, suffix heap size=139976,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=150 2024-11-19T17:16:15,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-19T17:16:15,501 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6bc673cb3ab3290d1b783693a45a9bb#A#compaction#520 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:16:15,502 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/69908ceb18234adf8e5d9dce96b21e35 is 175, key is test_row_0/A:col10/1732036573067/Put/seqid=0 2024-11-19T17:16:15,504 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/4199c00a41ce42dcab467923ee4cd761 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/4199c00a41ce42dcab467923ee4cd761 2024-11-19T17:16:15,508 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a6bc673cb3ab3290d1b783693a45a9bb/B of a6bc673cb3ab3290d1b783693a45a9bb into 4199c00a41ce42dcab467923ee4cd761(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:16:15,508 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:15,508 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., storeName=a6bc673cb3ab3290d1b783693a45a9bb/B, priority=12, startTime=1732036575082; duration=0sec 2024-11-19T17:16:15,508 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:16:15,508 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6bc673cb3ab3290d1b783693a45a9bb:B 2024-11-19T17:16:15,508 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:16:15,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742441_1617 (size=31093) 2024-11-19T17:16:15,510 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:16:15,511 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): a6bc673cb3ab3290d1b783693a45a9bb/C is initiating minor compaction (all files) 2024-11-19T17:16:15,511 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6bc673cb3ab3290d1b783693a45a9bb/C in TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:15,511 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/ac84d178d35d4a15a94bae66f890db1d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/d6e9b245ec364c89b91069eaec402c4a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/a506325c528d46aa95bae864101bde45, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/fd389427fe1f40328f013ace7b84111c] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp, totalSize=46.9 K 2024-11-19T17:16:15,511 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting ac84d178d35d4a15a94bae66f890db1d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732036570139 2024-11-19T17:16:15,511 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting d6e9b245ec364c89b91069eaec402c4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732036570206 2024-11-19T17:16:15,512 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting a506325c528d46aa95bae864101bde45, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732036570847 2024-11-19T17:16:15,512 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting fd389427fe1f40328f013ace7b84111c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732036573067 2024-11-19T17:16:15,515 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/69908ceb18234adf8e5d9dce96b21e35 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/69908ceb18234adf8e5d9dce96b21e35 2024-11-19T17:16:15,521 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6bc673cb3ab3290d1b783693a45a9bb#C#compaction#525 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:16:15,522 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/0f98c9a4e64c4a9cabe51c8b12f6c346 is 50, key is test_row_0/C:col10/1732036573067/Put/seqid=0 2024-11-19T17:16:15,522 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a6bc673cb3ab3290d1b783693a45a9bb/A of a6bc673cb3ab3290d1b783693a45a9bb into 69908ceb18234adf8e5d9dce96b21e35(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:16:15,523 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:15,523 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., storeName=a6bc673cb3ab3290d1b783693a45a9bb/A, priority=12, startTime=1732036575082; duration=0sec 2024-11-19T17:16:15,523 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:15,523 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6bc673cb3ab3290d1b783693a45a9bb:A 2024-11-19T17:16:15,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742442_1618 (size=12139) 2024-11-19T17:16:15,528 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/0f98c9a4e64c4a9cabe51c8b12f6c346 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/0f98c9a4e64c4a9cabe51c8b12f6c346 2024-11-19T17:16:15,532 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a6bc673cb3ab3290d1b783693a45a9bb/C of a6bc673cb3ab3290d1b783693a45a9bb into 0f98c9a4e64c4a9cabe51c8b12f6c346(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:16:15,532 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:15,533 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., storeName=a6bc673cb3ab3290d1b783693a45a9bb/C, priority=12, startTime=1732036575082; duration=0sec 2024-11-19T17:16:15,533 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:15,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:15,533 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6bc673cb3ab3290d1b783693a45a9bb:C 2024-11-19T17:16:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34398 deadline: 1732036635524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:15,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036635526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:15,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732036575500, max timestamp=1732036575531Num uniques 1136; 2024-11-19T17:16:15,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-11-19T17:16:15,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732036575500, max timestamp=1732036575531Num uniques 1136; 2024-11-19T17:16:15,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-11-19T17:16:15,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732036575500, max timestamp=1732036575531Num uniques 1009; 2024-11-19T17:16:15,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8516, suffix heap size=132904,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=186 2024-11-19T17:16:15,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:15,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:15,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732036575567, max timestamp=1732036575582Num uniques 1045; 2024-11-19T17:16:15,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-19T17:16:15,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732036575567, max timestamp=1732036575582Num uniques 1045; 2024-11-19T17:16:15,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-19T17:16:15,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732036575567, max timestamp=1732036575582Num uniques 1045; 2024-11-19T17:16:15,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-19T17:16:15,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T17:16:15,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:15,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036635619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:15,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732036575599, max timestamp=1732036575619Num uniques 1054; 2024-11-19T17:16:15,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-19T17:16:15,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:15,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732036575599, max timestamp=1732036575619Num uniques 1054; 2024-11-19T17:16:15,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-19T17:16:15,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732036575599, max timestamp=1732036575619Num uniques 1054; 2024-11-19T17:16:15,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-19T17:16:15,652 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/e246b35c30eb46b787c366447c0985a8 2024-11-19T17:16:15,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/1c6d923ffb1c4ce481f3da7650447edd as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/1c6d923ffb1c4ce481f3da7650447edd 2024-11-19T17:16:15,659 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/1c6d923ffb1c4ce481f3da7650447edd, entries=100, sequenceid=106, filesize=21.8 K 2024-11-19T17:16:15,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/8e5eff5b73fa4149a4cd3cb7723db4bc as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/8e5eff5b73fa4149a4cd3cb7723db4bc 2024-11-19T17:16:15,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34398 deadline: 1732036635657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:15,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:15,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036635657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:15,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:15,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036635658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:15,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:15,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:15,665 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,665 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:15,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732036575654, max timestamp=1732036575662Num uniques 1063; 2024-11-19T17:16:15,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-19T17:16:15,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732036575654, max timestamp=1732036575662Num uniques 1063; 2024-11-19T17:16:15,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732036575654, max timestamp=1732036575662Num uniques 1063; 2024-11-19T17:16:15,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-19T17:16:15,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-19T17:16:15,667 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/8e5eff5b73fa4149a4cd3cb7723db4bc, entries=100, sequenceid=106, filesize=9.4 K 2024-11-19T17:16:15,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/e246b35c30eb46b787c366447c0985a8 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/e246b35c30eb46b787c366447c0985a8 2024-11-19T17:16:15,671 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/e246b35c30eb46b787c366447c0985a8, entries=100, sequenceid=106, filesize=9.4 K 2024-11-19T17:16:15,672 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=33.54 KB/34350 for a6bc673cb3ab3290d1b783693a45a9bb in 461ms, sequenceid=106, compaction requested=false 2024-11-19T17:16:15,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:15,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:15,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-19T17:16:15,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-19T17:16:15,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-19T17:16:15,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2270 sec 2024-11-19T17:16:15,677 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.2310 sec 2024-11-19T17:16:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:15,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6bc673cb3ab3290d1b783693a45a9bb 3/3 column families, dataSize=53.67 KB heapSize=145.55 KB 2024-11-19T17:16:15,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=A 2024-11-19T17:16:15,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:16:15,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=B 2024-11-19T17:16:15,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:16:15,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=C 2024-11-19T17:16:15,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:16:15,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c2c00e5c5cc946c79ff1e2c5dfc6de83_a6bc673cb3ab3290d1b783693a45a9bb is 50, key is test_row_0/A:col10/1732036575655/Put/seqid=0 2024-11-19T17:16:15,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742443_1619 (size=14794) 2024-11-19T17:16:15,691 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:15,694 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c2c00e5c5cc946c79ff1e2c5dfc6de83_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c2c00e5c5cc946c79ff1e2c5dfc6de83_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:15,695 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/bfefa39f18db45b98aac0516d66b80f5, store: [table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:15,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/bfefa39f18db45b98aac0516d66b80f5 is 175, key is test_row_0/A:col10/1732036575655/Put/seqid=0 2024-11-19T17:16:15,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742444_1620 (size=39749) 2024-11-19T17:16:15,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036575720, max timestamp=1732036575749Num uniques 859; 2024-11-19T17:16:15,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036575720, max timestamp=1732036575749Num uniques 859; 2024-11-19T17:16:15,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036575720, max timestamp=1732036575749Num uniques 859; 2024-11-19T17:16:15,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:16:15,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:16:15,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:16:15,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:15,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036635779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:15,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:15,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34398 deadline: 1732036635781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:15,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:15,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:15,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:15,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036635782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:15,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:15,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:15,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34418 deadline: 1732036635783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:15,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:15,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036635784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:15,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:15,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036635894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:15,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:15,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:15,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34398 deadline: 1732036635894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:15,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036635894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:15,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34418 deadline: 1732036635894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:15,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036635894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:15,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:15,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:16,099 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/bfefa39f18db45b98aac0516d66b80f5 2024-11-19T17:16:16,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/e4b8fe5f444f46e4a8a3cbf07ea7d187 is 50, key is test_row_0/B:col10/1732036575655/Put/seqid=0 2024-11-19T17:16:16,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:16,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036636104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:16,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:16,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036636104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:16,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:16,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34398 deadline: 1732036636104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:16,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:16,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742445_1621 (size=12151) 2024-11-19T17:16:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:16,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34418 deadline: 1732036636104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:16,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:16,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036636105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:16,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036636413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:16,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:16,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036636413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:16,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:16,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:16,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:16,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:16,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:16,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:16,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34398 deadline: 1732036636414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:16,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:16,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:16,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34418 deadline: 1732036636414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:16,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:16,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:16,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:16,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:16,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036636414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:16,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:16,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:16,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:16,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:16,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:16,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:16,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/e4b8fe5f444f46e4a8a3cbf07ea7d187 2024-11-19T17:16:16,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/ef242198db4a488ea84989408202ef67 is 50, key is test_row_0/C:col10/1732036575655/Put/seqid=0 2024-11-19T17:16:16,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742446_1622 (size=12151) 2024-11-19T17:16:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T17:16:16,614 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:16:16,614 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:16:16,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-19T17:16:16,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-19T17:16:16,616 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:16:16,616 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:16:16,616 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:16:16,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-19T17:16:16,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-19T17:16:16,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:16,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:16,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:16,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:16,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:16,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:16,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/ef242198db4a488ea84989408202ef67 2024-11-19T17:16:16,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-19T17:16:16,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/bfefa39f18db45b98aac0516d66b80f5 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/bfefa39f18db45b98aac0516d66b80f5 2024-11-19T17:16:16,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:16,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:16,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:16,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:16,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:16,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:16,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/bfefa39f18db45b98aac0516d66b80f5, entries=200, sequenceid=237, filesize=38.8 K 2024-11-19T17:16:16,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/e4b8fe5f444f46e4a8a3cbf07ea7d187 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/e4b8fe5f444f46e4a8a3cbf07ea7d187 2024-11-19T17:16:16,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:16,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036636924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:16,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:16,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:16,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036636924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:16,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:16,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:16,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:16,928 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:16,928 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:16,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34398 deadline: 1732036636924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:16,928 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:16,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:16,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:16,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:16,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34418 deadline: 1732036636925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:16,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:16,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:16,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:16,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036636925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:16,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:16,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:16,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:16,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:16,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:16,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/e4b8fe5f444f46e4a8a3cbf07ea7d187, entries=150, sequenceid=237, filesize=11.9 K 2024-11-19T17:16:16,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/ef242198db4a488ea84989408202ef67 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/ef242198db4a488ea84989408202ef67 2024-11-19T17:16:16,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-19T17:16:16,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/ef242198db4a488ea84989408202ef67, entries=150, sequenceid=237, filesize=11.9 K 2024-11-19T17:16:16,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.50 KB/148992, currentSize=134.18 KB/137400 for a6bc673cb3ab3290d1b783693a45a9bb in 1258ms, sequenceid=237, compaction requested=true 2024-11-19T17:16:16,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:16,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6bc673cb3ab3290d1b783693a45a9bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:16:16,938 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:16:16,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:16,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6bc673cb3ab3290d1b783693a45a9bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:16:16,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:16,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6bc673cb3ab3290d1b783693a45a9bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:16:16,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:16:16,938 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:16:16,938 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93203 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:16:16,938 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33947 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:16:16,938 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): a6bc673cb3ab3290d1b783693a45a9bb/A is initiating minor compaction (all files) 2024-11-19T17:16:16,938 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): a6bc673cb3ab3290d1b783693a45a9bb/B is initiating minor compaction (all files) 2024-11-19T17:16:16,938 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6bc673cb3ab3290d1b783693a45a9bb/A in TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:16,938 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6bc673cb3ab3290d1b783693a45a9bb/B in TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:16,939 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/69908ceb18234adf8e5d9dce96b21e35, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/1c6d923ffb1c4ce481f3da7650447edd, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/bfefa39f18db45b98aac0516d66b80f5] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp, totalSize=91.0 K 2024-11-19T17:16:16,939 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/4199c00a41ce42dcab467923ee4cd761, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/8e5eff5b73fa4149a4cd3cb7723db4bc, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/e4b8fe5f444f46e4a8a3cbf07ea7d187] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp, totalSize=33.2 K 2024-11-19T17:16:16,939 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:16,939 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. files: [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/69908ceb18234adf8e5d9dce96b21e35, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/1c6d923ffb1c4ce481f3da7650447edd, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/bfefa39f18db45b98aac0516d66b80f5] 2024-11-19T17:16:16,940 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 4199c00a41ce42dcab467923ee4cd761, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732036573067 2024-11-19T17:16:16,940 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69908ceb18234adf8e5d9dce96b21e35, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732036573067 2024-11-19T17:16:16,940 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e5eff5b73fa4149a4cd3cb7723db4bc, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1732036574226 2024-11-19T17:16:16,940 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c6d923ffb1c4ce481f3da7650447edd, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1732036574226 2024-11-19T17:16:16,940 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting e4b8fe5f444f46e4a8a3cbf07ea7d187, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732036575655 2024-11-19T17:16:16,940 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfefa39f18db45b98aac0516d66b80f5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732036575655 2024-11-19T17:16:16,945 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:16,945 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6bc673cb3ab3290d1b783693a45a9bb#B#compaction#529 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:16:16,946 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/6b12cf53a5964a55afe9e7572c64c1fc is 50, key is test_row_0/B:col10/1732036575655/Put/seqid=0 2024-11-19T17:16:16,959 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411197987dec360044573aa590ce397149409_a6bc673cb3ab3290d1b783693a45a9bb store=[table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:16,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742447_1623 (size=12391) 2024-11-19T17:16:16,961 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411197987dec360044573aa590ce397149409_a6bc673cb3ab3290d1b783693a45a9bb, store=[table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:16,962 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197987dec360044573aa590ce397149409_a6bc673cb3ab3290d1b783693a45a9bb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:16,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742448_1624 (size=4469) 2024-11-19T17:16:16,966 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/6b12cf53a5964a55afe9e7572c64c1fc as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/6b12cf53a5964a55afe9e7572c64c1fc 2024-11-19T17:16:16,969 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6bc673cb3ab3290d1b783693a45a9bb#A#compaction#530 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:16:16,970 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/b31a419094e9403c920dec72cd297cd8 is 175, key is test_row_0/A:col10/1732036575655/Put/seqid=0 2024-11-19T17:16:16,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742449_1625 (size=31345) 2024-11-19T17:16:16,973 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6bc673cb3ab3290d1b783693a45a9bb/B of a6bc673cb3ab3290d1b783693a45a9bb into 6b12cf53a5964a55afe9e7572c64c1fc(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:16:16,973 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:16,973 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., storeName=a6bc673cb3ab3290d1b783693a45a9bb/B, priority=13, startTime=1732036576938; duration=0sec 2024-11-19T17:16:16,973 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:16:16,973 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6bc673cb3ab3290d1b783693a45a9bb:B 2024-11-19T17:16:16,973 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:16:16,974 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33947 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:16:16,974 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): a6bc673cb3ab3290d1b783693a45a9bb/C is initiating minor compaction (all files) 2024-11-19T17:16:16,974 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6bc673cb3ab3290d1b783693a45a9bb/C in TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:16,975 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/0f98c9a4e64c4a9cabe51c8b12f6c346, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/e246b35c30eb46b787c366447c0985a8, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/ef242198db4a488ea84989408202ef67] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp, totalSize=33.2 K 2024-11-19T17:16:16,975 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f98c9a4e64c4a9cabe51c8b12f6c346, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732036573067 2024-11-19T17:16:16,975 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting e246b35c30eb46b787c366447c0985a8, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1732036574226 2024-11-19T17:16:16,976 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting ef242198db4a488ea84989408202ef67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732036575655 2024-11-19T17:16:16,977 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/b31a419094e9403c920dec72cd297cd8 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/b31a419094e9403c920dec72cd297cd8 2024-11-19T17:16:16,981 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6bc673cb3ab3290d1b783693a45a9bb/A of a6bc673cb3ab3290d1b783693a45a9bb into b31a419094e9403c920dec72cd297cd8(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:16:16,981 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:16,981 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., storeName=a6bc673cb3ab3290d1b783693a45a9bb/A, priority=13, startTime=1732036576937; duration=0sec 2024-11-19T17:16:16,981 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:16,981 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6bc673cb3ab3290d1b783693a45a9bb:A 2024-11-19T17:16:16,985 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6bc673cb3ab3290d1b783693a45a9bb#C#compaction#531 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:16:16,986 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/320ef2e2ee364c1ba19dc903edae35c8 is 50, key is test_row_0/C:col10/1732036575655/Put/seqid=0 2024-11-19T17:16:16,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742450_1626 (size=12391) 2024-11-19T17:16:17,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-19T17:16:17,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:17,075 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing a6bc673cb3ab3290d1b783693a45a9bb 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-19T17:16:17,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=A 2024-11-19T17:16:17,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:16:17,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=B 2024-11-19T17:16:17,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:16:17,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=C 2024-11-19T17:16:17,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:16:17,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119378811244e244188b7692c9d049ccd41_a6bc673cb3ab3290d1b783693a45a9bb is 50, key is test_row_0/A:col10/1732036575781/Put/seqid=0 2024-11-19T17:16:17,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742451_1627 (size=12454) 2024-11-19T17:16:17,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-19T17:16:17,398 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/320ef2e2ee364c1ba19dc903edae35c8 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/320ef2e2ee364c1ba19dc903edae35c8 2024-11-19T17:16:17,402 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6bc673cb3ab3290d1b783693a45a9bb/C of a6bc673cb3ab3290d1b783693a45a9bb into 320ef2e2ee364c1ba19dc903edae35c8(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:16:17,402 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:17,402 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., storeName=a6bc673cb3ab3290d1b783693a45a9bb/C, priority=13, startTime=1732036576938; duration=0sec 2024-11-19T17:16:17,402 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:17,402 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6bc673cb3ab3290d1b783693a45a9bb:C 2024-11-19T17:16:17,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:17,490 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119378811244e244188b7692c9d049ccd41_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119378811244e244188b7692c9d049ccd41_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:17,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/dea6954c8cd04918aa80a6a8803614bf, store: [table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:17,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/dea6954c8cd04918aa80a6a8803614bf is 175, key is test_row_0/A:col10/1732036575781/Put/seqid=0 2024-11-19T17:16:17,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742452_1628 (size=31255) 2024-11-19T17:16:17,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-19T17:16:17,895 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/dea6954c8cd04918aa80a6a8803614bf 2024-11-19T17:16:17,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/6889ecc63cd74493b4ed9a007e777cfb is 50, key is test_row_0/B:col10/1732036575781/Put/seqid=0 2024-11-19T17:16:17,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742453_1629 (size=12301) 2024-11-19T17:16:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:17,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:17,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34398 deadline: 1732036637943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:17,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34418 deadline: 1732036637943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:17,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036637944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:17,950 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_1' in a6bc673cb3ab3290d1b783693a45a9bb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:16:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:17,950 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_1' in a6bc673cb3ab3290d1b783693a45a9bb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:16:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:17,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:17,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:17,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036637949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:17,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036637949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:17,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:17,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:17,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:17,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:17,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:17,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:18,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036638053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:18,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:18,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:18,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:18,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:18,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036638063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:18,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:18,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:18,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:18,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:18,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036638064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:18,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:18,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:18,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:18,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036638264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:18,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:18,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:18,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:18,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036638274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:18,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:18,278 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:18,278 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036638274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:18,278 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:18,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:18,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:18,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:18,305 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/6889ecc63cd74493b4ed9a007e777cfb 2024-11-19T17:16:18,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/adbb3e84c19e4ccaac318ae0d464f3c0 is 50, key is test_row_0/C:col10/1732036575781/Put/seqid=0 2024-11-19T17:16:18,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742454_1630 (size=12301) 2024-11-19T17:16:18,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036638573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:18,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:18,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:18,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:18,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036638583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:18,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:18,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:18,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:18,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:18,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036638583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:18,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:18,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:18,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:18,716 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/adbb3e84c19e4ccaac318ae0d464f3c0 2024-11-19T17:16:18,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/dea6954c8cd04918aa80a6a8803614bf as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/dea6954c8cd04918aa80a6a8803614bf 2024-11-19T17:16:18,723 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/dea6954c8cd04918aa80a6a8803614bf, entries=150, sequenceid=275, filesize=30.5 K 2024-11-19T17:16:18,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/6889ecc63cd74493b4ed9a007e777cfb as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/6889ecc63cd74493b4ed9a007e777cfb 2024-11-19T17:16:18,726 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/6889ecc63cd74493b4ed9a007e777cfb, entries=150, sequenceid=275, filesize=12.0 K 2024-11-19T17:16:18,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/adbb3e84c19e4ccaac318ae0d464f3c0 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/adbb3e84c19e4ccaac318ae0d464f3c0 2024-11-19T17:16:18,729 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/adbb3e84c19e4ccaac318ae0d464f3c0, entries=150, sequenceid=275, filesize=12.0 K 2024-11-19T17:16:18,730 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for a6bc673cb3ab3290d1b783693a45a9bb in 1655ms, sequenceid=275, compaction requested=false 2024-11-19T17:16:18,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:18,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:18,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-19T17:16:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-19T17:16:18,734 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-19T17:16:18,734 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1160 sec 2024-11-19T17:16:18,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 2.1200 sec 2024-11-19T17:16:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-19T17:16:18,764 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:16:18,765 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:16:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-19T17:16:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-19T17:16:18,766 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:16:18,767 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:16:18,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:16:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-19T17:16:18,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-19T17:16:18,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:18,919 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing a6bc673cb3ab3290d1b783693a45a9bb 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T17:16:18,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=A 2024-11-19T17:16:18,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:18,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=B 2024-11-19T17:16:18,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:18,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=C 2024-11-19T17:16:18,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:18,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c121dcac99eb44a1bc757d618982d2d2_a6bc673cb3ab3290d1b783693a45a9bb is 50, key is test_row_0/A:col10/1732036577948/Put/seqid=0 2024-11-19T17:16:18,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742455_1631 (size=12454) 2024-11-19T17:16:18,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,932 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c121dcac99eb44a1bc757d618982d2d2_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c121dcac99eb44a1bc757d618982d2d2_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:18,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/abd5ca30f7c247e0a6ed71ecb19b4c9d, store: [table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:18,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/abd5ca30f7c247e0a6ed71ecb19b4c9d is 175, key is test_row_0/A:col10/1732036577948/Put/seqid=0 2024-11-19T17:16:18,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742456_1632 (size=31255) 2024-11-19T17:16:18,938 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/abd5ca30f7c247e0a6ed71ecb19b4c9d 2024-11-19T17:16:18,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/a1c5e88a989142079c7cc937eda9e0ec is 50, key is test_row_0/B:col10/1732036577948/Put/seqid=0 2024-11-19T17:16:18,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742457_1633 (size=12301) 2024-11-19T17:16:18,947 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/a1c5e88a989142079c7cc937eda9e0ec 2024-11-19T17:16:18,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/6f2228602b9343f4a574e85b39491889 is 50, key is test_row_0/C:col10/1732036577948/Put/seqid=0 2024-11-19T17:16:18,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742458_1634 (size=12301) 2024-11-19T17:16:18,957 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/6f2228602b9343f4a574e85b39491889 2024-11-19T17:16:18,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/abd5ca30f7c247e0a6ed71ecb19b4c9d as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/abd5ca30f7c247e0a6ed71ecb19b4c9d 2024-11-19T17:16:18,964 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/abd5ca30f7c247e0a6ed71ecb19b4c9d, entries=150, sequenceid=287, filesize=30.5 K 2024-11-19T17:16:18,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/a1c5e88a989142079c7cc937eda9e0ec as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/a1c5e88a989142079c7cc937eda9e0ec 2024-11-19T17:16:18,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,967 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/a1c5e88a989142079c7cc937eda9e0ec, entries=150, sequenceid=287, filesize=12.0 K 2024-11-19T17:16:18,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/6f2228602b9343f4a574e85b39491889 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/6f2228602b9343f4a574e85b39491889 2024-11-19T17:16:18,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,971 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/6f2228602b9343f4a574e85b39491889, entries=150, sequenceid=287, filesize=12.0 K 2024-11-19T17:16:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,972 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for a6bc673cb3ab3290d1b783693a45a9bb in 53ms, sequenceid=287, compaction requested=true 2024-11-19T17:16:18,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:18,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:18,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-19T17:16:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-19T17:16:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,975 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-19T17:16:18,975 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 206 msec 2024-11-19T17:16:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,977 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 211 msec 2024-11-19T17:16:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-19T17:16:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,084 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:16:19,084 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:16:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-19T17:16:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T17:16:19,086 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:16:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,087 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:16:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,087 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:16:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6bc673cb3ab3290d1b783693a45a9bb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T17:16:19,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=A 2024-11-19T17:16:19,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:19,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=B 2024-11-19T17:16:19,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:19,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=C 2024-11-19T17:16:19,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ae625b75aa114b63b4a5e70fc933787e_a6bc673cb3ab3290d1b783693a45a9bb is 50, key is test_row_0/A:col10/1732036579108/Put/seqid=0 2024-11-19T17:16:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742459_1635 (size=22618) 2024-11-19T17:16:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036579154, max timestamp=1732036579162Num uniques 859; 2024-11-19T17:16:19,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:16:19,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036579154, max timestamp=1732036579162Num uniques 859; 2024-11-19T17:16:19,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036579154, max timestamp=1732036579162Num uniques 859; 2024-11-19T17:16:19,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:16:19,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:16:19,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T17:16:19,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:19,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036639195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:19,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:19,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036639196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:19,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:19,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036639200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:19,206 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:19,206 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:19,206 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:19,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-19T17:16:19,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:19,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:19,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:19,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:19,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:19,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:19,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:19,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036639303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:19,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:19,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:19,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036639304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:19,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:19,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036639314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:19,319 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:19,319 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:19,319 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:19,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-19T17:16:19,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:19,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:19,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:19,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:19,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:19,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:19,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T17:16:19,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:19,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036639513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:19,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:19,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:19,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:19,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036639513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:19,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:19,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:19,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:19,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:19,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:19,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036639523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:19,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:19,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:19,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:19,535 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:19,538 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ae625b75aa114b63b4a5e70fc933787e_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ae625b75aa114b63b4a5e70fc933787e_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:19,538 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/75af5866857444ea9ae2a4c677ffe516, store: [table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:19,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/75af5866857444ea9ae2a4c677ffe516 is 175, key is test_row_0/A:col10/1732036579108/Put/seqid=0 2024-11-19T17:16:19,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742460_1636 (size=66023) 2024-11-19T17:16:19,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-19T17:16:19,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:19,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:19,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:19,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:19,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:19,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:19,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-19T17:16:19,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:19,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:19,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:19,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:19,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:19,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:19,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T17:16:19,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:19,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036639824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:19,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:19,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036639824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:19,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:19,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036639833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:19,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-19T17:16:19,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:19,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:19,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:19,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:19,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:19,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:19,943 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=297, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/75af5866857444ea9ae2a4c677ffe516 2024-11-19T17:16:19,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/6a04037055f04162b7571296360e3c1f is 50, key is test_row_0/B:col10/1732036579108/Put/seqid=0 2024-11-19T17:16:19,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742461_1637 (size=12301) 2024-11-19T17:16:19,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:19,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34418 deadline: 1732036639963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:19,967 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_1' in a6bc673cb3ab3290d1b783693a45a9bb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:16:19,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:19,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:19,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:19,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:19,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34398 deadline: 1732036639973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:19,978 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_1' in a6bc673cb3ab3290d1b783693a45a9bb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:16:19,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:19,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:19,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:20,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-19T17:16:20,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:20,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:20,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:20,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:20,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:20,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:20,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-19T17:16:20,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:20,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:20,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:20,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:20,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:20,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:20,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T17:16:20,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-19T17:16:20,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:20,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:20,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:20,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:20,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:20,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:20,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036640333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:20,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036640333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:20,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:20,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:20,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:20,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036640344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:20,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/6a04037055f04162b7571296360e3c1f 2024-11-19T17:16:20,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/e960304e5e794d1689a07d16fe7d76b1 is 50, key is test_row_0/C:col10/1732036579108/Put/seqid=0 2024-11-19T17:16:20,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742462_1638 (size=12301) 2024-11-19T17:16:20,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-19T17:16:20,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:20,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:20,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:20,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:20,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:20,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-19T17:16:20,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:20,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:20,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:20,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:20,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:20,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:20,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-19T17:16:20,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:20,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/e960304e5e794d1689a07d16fe7d76b1 2024-11-19T17:16:20,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:20,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:20,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:20,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:20,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:20,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/75af5866857444ea9ae2a4c677ffe516 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/75af5866857444ea9ae2a4c677ffe516 2024-11-19T17:16:20,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/75af5866857444ea9ae2a4c677ffe516, entries=350, sequenceid=297, filesize=64.5 K 2024-11-19T17:16:20,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/6a04037055f04162b7571296360e3c1f as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/6a04037055f04162b7571296360e3c1f 2024-11-19T17:16:20,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/6a04037055f04162b7571296360e3c1f, entries=150, sequenceid=297, filesize=12.0 K 2024-11-19T17:16:20,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/e960304e5e794d1689a07d16fe7d76b1 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/e960304e5e794d1689a07d16fe7d76b1 2024-11-19T17:16:20,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/e960304e5e794d1689a07d16fe7d76b1, entries=150, sequenceid=297, filesize=12.0 K 2024-11-19T17:16:20,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a6bc673cb3ab3290d1b783693a45a9bb in 1658ms, sequenceid=297, compaction requested=true 2024-11-19T17:16:20,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:20,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6bc673cb3ab3290d1b783693a45a9bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:16:20,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:20,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6bc673cb3ab3290d1b783693a45a9bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:16:20,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:20,783 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:16:20,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6bc673cb3ab3290d1b783693a45a9bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:16:20,783 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:16:20,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:16:20,784 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49294 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:16:20,784 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 159878 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:16:20,784 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): a6bc673cb3ab3290d1b783693a45a9bb/B is initiating minor compaction (all files) 2024-11-19T17:16:20,784 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): a6bc673cb3ab3290d1b783693a45a9bb/A is initiating minor compaction (all files) 2024-11-19T17:16:20,784 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6bc673cb3ab3290d1b783693a45a9bb/B in TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:20,784 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6bc673cb3ab3290d1b783693a45a9bb/A in TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:20,784 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/6b12cf53a5964a55afe9e7572c64c1fc, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/6889ecc63cd74493b4ed9a007e777cfb, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/a1c5e88a989142079c7cc937eda9e0ec, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/6a04037055f04162b7571296360e3c1f] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp, totalSize=48.1 K 2024-11-19T17:16:20,784 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/b31a419094e9403c920dec72cd297cd8, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/dea6954c8cd04918aa80a6a8803614bf, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/abd5ca30f7c247e0a6ed71ecb19b4c9d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/75af5866857444ea9ae2a4c677ffe516] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp, totalSize=156.1 K 2024-11-19T17:16:20,784 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:20,784 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. files: [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/b31a419094e9403c920dec72cd297cd8, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/dea6954c8cd04918aa80a6a8803614bf, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/abd5ca30f7c247e0a6ed71ecb19b4c9d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/75af5866857444ea9ae2a4c677ffe516] 2024-11-19T17:16:20,784 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b12cf53a5964a55afe9e7572c64c1fc, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732036575655 2024-11-19T17:16:20,784 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting b31a419094e9403c920dec72cd297cd8, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732036575655 2024-11-19T17:16:20,784 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 6889ecc63cd74493b4ed9a007e777cfb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732036575780 2024-11-19T17:16:20,785 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting dea6954c8cd04918aa80a6a8803614bf, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732036575780 2024-11-19T17:16:20,785 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting a1c5e88a989142079c7cc937eda9e0ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732036577938 2024-11-19T17:16:20,785 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting abd5ca30f7c247e0a6ed71ecb19b4c9d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732036577938 2024-11-19T17:16:20,785 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a04037055f04162b7571296360e3c1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732036579108 2024-11-19T17:16:20,785 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75af5866857444ea9ae2a4c677ffe516, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732036579108 2024-11-19T17:16:20,790 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:20,792 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411197f052b736de54eea883b83c83643ec21_a6bc673cb3ab3290d1b783693a45a9bb store=[table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:20,792 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6bc673cb3ab3290d1b783693a45a9bb#B#compaction#542 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:16:20,793 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/74c0ce18a2434242af71b062319b5cc0 is 50, key is test_row_0/B:col10/1732036579108/Put/seqid=0 2024-11-19T17:16:20,794 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411197f052b736de54eea883b83c83643ec21_a6bc673cb3ab3290d1b783693a45a9bb, store=[table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:20,794 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197f052b736de54eea883b83c83643ec21_a6bc673cb3ab3290d1b783693a45a9bb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:20,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742463_1639 (size=12677) 2024-11-19T17:16:20,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742464_1640 (size=4469) 2024-11-19T17:16:20,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-19T17:16:20,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:20,921 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing a6bc673cb3ab3290d1b783693a45a9bb 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-19T17:16:20,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=A 2024-11-19T17:16:20,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:16:20,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=B 2024-11-19T17:16:20,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:16:20,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=C 2024-11-19T17:16:20,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:16:20,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197e479f695ebe4fe6adaa534ebfa24fdb_a6bc673cb3ab3290d1b783693a45a9bb is 50, key is test_row_0/A:col10/1732036579194/Put/seqid=0 2024-11-19T17:16:20,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742465_1641 (size=12454) 2024-11-19T17:16:21,200 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/74c0ce18a2434242af71b062319b5cc0 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/74c0ce18a2434242af71b062319b5cc0 2024-11-19T17:16:21,202 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6bc673cb3ab3290d1b783693a45a9bb#A#compaction#541 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:16:21,203 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/a06751b29faa4f47b3d332e541d36f4c is 175, key is test_row_0/A:col10/1732036579108/Put/seqid=0 2024-11-19T17:16:21,205 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a6bc673cb3ab3290d1b783693a45a9bb/B of a6bc673cb3ab3290d1b783693a45a9bb into 74c0ce18a2434242af71b062319b5cc0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:16:21,205 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:21,205 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., storeName=a6bc673cb3ab3290d1b783693a45a9bb/B, priority=12, startTime=1732036580783; duration=0sec 2024-11-19T17:16:21,205 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:16:21,205 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6bc673cb3ab3290d1b783693a45a9bb:B 2024-11-19T17:16:21,205 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T17:16:21,206 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49294 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T17:16:21,206 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): a6bc673cb3ab3290d1b783693a45a9bb/C is initiating minor compaction (all files) 2024-11-19T17:16:21,206 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6bc673cb3ab3290d1b783693a45a9bb/C in TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:21,207 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/320ef2e2ee364c1ba19dc903edae35c8, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/adbb3e84c19e4ccaac318ae0d464f3c0, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/6f2228602b9343f4a574e85b39491889, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/e960304e5e794d1689a07d16fe7d76b1] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp, totalSize=48.1 K 2024-11-19T17:16:21,207 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 320ef2e2ee364c1ba19dc903edae35c8, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732036575655 2024-11-19T17:16:21,207 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting adbb3e84c19e4ccaac318ae0d464f3c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732036575780 2024-11-19T17:16:21,208 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f2228602b9343f4a574e85b39491889, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732036577938 2024-11-19T17:16:21,208 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting e960304e5e794d1689a07d16fe7d76b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732036579108 2024-11-19T17:16:21,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742466_1642 (size=31631) 2024-11-19T17:16:21,219 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6bc673cb3ab3290d1b783693a45a9bb#C#compaction#544 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:16:21,219 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/e6393ae902cb419aacb06bbd123f74a7 is 50, key is test_row_0/C:col10/1732036579108/Put/seqid=0 2024-11-19T17:16:21,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742467_1643 (size=12677) 2024-11-19T17:16:21,227 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/a06751b29faa4f47b3d332e541d36f4c as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/a06751b29faa4f47b3d332e541d36f4c 2024-11-19T17:16:21,231 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a6bc673cb3ab3290d1b783693a45a9bb/A of a6bc673cb3ab3290d1b783693a45a9bb into a06751b29faa4f47b3d332e541d36f4c(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:16:21,231 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:21,231 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., storeName=a6bc673cb3ab3290d1b783693a45a9bb/A, priority=12, startTime=1732036580782; duration=0sec 2024-11-19T17:16:21,231 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:21,231 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6bc673cb3ab3290d1b783693a45a9bb:A 2024-11-19T17:16:21,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T17:16:21,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,341 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197e479f695ebe4fe6adaa534ebfa24fdb_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197e479f695ebe4fe6adaa534ebfa24fdb_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:21,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/338e48e9869f40b98904f04e3dd1a1bf, store: [table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:21,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/338e48e9869f40b98904f04e3dd1a1bf is 175, key is test_row_0/A:col10/1732036579194/Put/seqid=0 2024-11-19T17:16:21,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742468_1644 (size=31255) 2024-11-19T17:16:21,351 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/338e48e9869f40b98904f04e3dd1a1bf 2024-11-19T17:16:21,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:21,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:21,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/61f399c154bb4ec4a8d7a7b07ed5577f is 50, key is test_row_0/B:col10/1732036579194/Put/seqid=0 2024-11-19T17:16:21,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742469_1645 (size=12301) 2024-11-19T17:16:21,361 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/61f399c154bb4ec4a8d7a7b07ed5577f 2024-11-19T17:16:21,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/384a775e68074ec68ce5ccf941754475 is 50, key is test_row_0/C:col10/1732036579194/Put/seqid=0 2024-11-19T17:16:21,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:21,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742470_1646 (size=12301) 2024-11-19T17:16:21,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036641365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:21,371 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/384a775e68074ec68ce5ccf941754475 2024-11-19T17:16:21,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:21,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:21,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:21,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:21,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036641368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:21,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:21,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036641369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:21,374 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:21,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/338e48e9869f40b98904f04e3dd1a1bf as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/338e48e9869f40b98904f04e3dd1a1bf 2024-11-19T17:16:21,377 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/338e48e9869f40b98904f04e3dd1a1bf, entries=150, sequenceid=334, filesize=30.5 K 2024-11-19T17:16:21,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/61f399c154bb4ec4a8d7a7b07ed5577f as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/61f399c154bb4ec4a8d7a7b07ed5577f 2024-11-19T17:16:21,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,381 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/61f399c154bb4ec4a8d7a7b07ed5577f, entries=150, sequenceid=334, filesize=12.0 K 2024-11-19T17:16:21,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/384a775e68074ec68ce5ccf941754475 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/384a775e68074ec68ce5ccf941754475 2024-11-19T17:16:21,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,384 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/384a775e68074ec68ce5ccf941754475, entries=150, sequenceid=334, filesize=12.0 K 2024-11-19T17:16:21,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,385 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for a6bc673cb3ab3290d1b783693a45a9bb in 464ms, sequenceid=334, compaction requested=false 2024-11-19T17:16:21,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:21,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:21,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-19T17:16:21,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-19T17:16:21,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,387 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-19T17:16:21,387 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2990 sec 2024-11-19T17:16:21,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,389 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 2.3030 sec 2024-11-19T17:16:21,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:21,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6bc673cb3ab3290d1b783693a45a9bb 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T17:16:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=A 2024-11-19T17:16:21,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:21,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=B 2024-11-19T17:16:21,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:21,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=C 2024-11-19T17:16:21,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:21,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ebd549b6998e41aca98298bc11ee9add_a6bc673cb3ab3290d1b783693a45a9bb is 50, key is test_row_0/A:col10/1732036581367/Put/seqid=0 2024-11-19T17:16:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742471_1647 (size=12454) 2024-11-19T17:16:21,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:21,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036641512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:21,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036581505, max timestamp=1732036581514Num uniques 859; 2024-11-19T17:16:21,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036581505, max timestamp=1732036581514Num uniques 859; 2024-11-19T17:16:21,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:16:21,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:16:21,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732036581505, max timestamp=1732036581514Num uniques 859; 2024-11-19T17:16:21,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T17:16:21,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:21,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:21,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:21,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:21,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036641553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:21,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:21,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036641559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:21,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:21,565 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:21,565 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:21,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:21,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036641623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:21,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:21,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:21,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:21,629 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/e6393ae902cb419aacb06bbd123f74a7 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/e6393ae902cb419aacb06bbd123f74a7 2024-11-19T17:16:21,632 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a6bc673cb3ab3290d1b783693a45a9bb/C of a6bc673cb3ab3290d1b783693a45a9bb into e6393ae902cb419aacb06bbd123f74a7(size=12.4 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:16:21,632 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:21,632 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., storeName=a6bc673cb3ab3290d1b783693a45a9bb/C, priority=12, startTime=1732036580783; duration=0sec 2024-11-19T17:16:21,632 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:21,632 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6bc673cb3ab3290d1b783693a45a9bb:C 2024-11-19T17:16:21,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:21,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036641664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:21,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:21,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:21,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:21,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:21,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036641673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:21,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:21,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:21,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:21,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036641834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:21,838 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:21,838 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:21,838 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:21,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:21,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036641874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:21,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:21,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:21,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:21,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036641883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:21,886 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:21,886 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:21,887 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:21,887 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:21,890 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ebd549b6998e41aca98298bc11ee9add_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ebd549b6998e41aca98298bc11ee9add_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:21,890 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/ea71352aba58407496d60804b06d8385, store: [table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:21,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/ea71352aba58407496d60804b06d8385 is 175, key is test_row_0/A:col10/1732036581367/Put/seqid=0 2024-11-19T17:16:21,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742472_1648 (size=31251) 2024-11-19T17:16:22,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036642143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:22,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:22,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:22,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:22,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036642184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:22,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:22,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:22,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:22,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036642194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:22,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:22,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:22,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:22,295 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=347, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/ea71352aba58407496d60804b06d8385 2024-11-19T17:16:22,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/5b00b513e788464cb53fdd872398fdd7 is 50, key is test_row_0/B:col10/1732036581367/Put/seqid=0 2024-11-19T17:16:22,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742473_1649 (size=9857) 2024-11-19T17:16:22,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:22,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036642653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:22,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:22,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:22,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:22,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036642694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:22,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:22,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:22,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:22,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/5b00b513e788464cb53fdd872398fdd7 2024-11-19T17:16:22,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036642704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:22,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/da8e30ae7e3e43ba8fd0eb127f8a8683 is 50, key is test_row_0/C:col10/1732036581367/Put/seqid=0 2024-11-19T17:16:22,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742474_1650 (size=9857) 2024-11-19T17:16:23,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/da8e30ae7e3e43ba8fd0eb127f8a8683 2024-11-19T17:16:23,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/ea71352aba58407496d60804b06d8385 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/ea71352aba58407496d60804b06d8385 2024-11-19T17:16:23,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/ea71352aba58407496d60804b06d8385, entries=150, sequenceid=347, filesize=30.5 K 2024-11-19T17:16:23,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/5b00b513e788464cb53fdd872398fdd7 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/5b00b513e788464cb53fdd872398fdd7 2024-11-19T17:16:23,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/5b00b513e788464cb53fdd872398fdd7, entries=100, sequenceid=347, filesize=9.6 K 2024-11-19T17:16:23,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/da8e30ae7e3e43ba8fd0eb127f8a8683 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/da8e30ae7e3e43ba8fd0eb127f8a8683 2024-11-19T17:16:23,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/da8e30ae7e3e43ba8fd0eb127f8a8683, entries=100, sequenceid=347, filesize=9.6 K 2024-11-19T17:16:23,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a6bc673cb3ab3290d1b783693a45a9bb in 1653ms, sequenceid=347, compaction requested=true 2024-11-19T17:16:23,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:23,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6bc673cb3ab3290d1b783693a45a9bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:16:23,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:23,130 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:16:23,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6bc673cb3ab3290d1b783693a45a9bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:16:23,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:23,131 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:16:23,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6bc673cb3ab3290d1b783693a45a9bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:16:23,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:16:23,131 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94137 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:16:23,131 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34835 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:16:23,131 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): a6bc673cb3ab3290d1b783693a45a9bb/A is initiating minor compaction (all files) 2024-11-19T17:16:23,131 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): a6bc673cb3ab3290d1b783693a45a9bb/B is initiating minor compaction (all files) 2024-11-19T17:16:23,131 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6bc673cb3ab3290d1b783693a45a9bb/A in TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:23,131 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6bc673cb3ab3290d1b783693a45a9bb/B in TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:23,131 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/a06751b29faa4f47b3d332e541d36f4c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/338e48e9869f40b98904f04e3dd1a1bf, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/ea71352aba58407496d60804b06d8385] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp, totalSize=91.9 K 2024-11-19T17:16:23,131 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/74c0ce18a2434242af71b062319b5cc0, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/61f399c154bb4ec4a8d7a7b07ed5577f, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/5b00b513e788464cb53fdd872398fdd7] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp, totalSize=34.0 K 2024-11-19T17:16:23,131 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:23,131 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. files: [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/a06751b29faa4f47b3d332e541d36f4c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/338e48e9869f40b98904f04e3dd1a1bf, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/ea71352aba58407496d60804b06d8385] 2024-11-19T17:16:23,132 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 74c0ce18a2434242af71b062319b5cc0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732036579108 2024-11-19T17:16:23,132 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting a06751b29faa4f47b3d332e541d36f4c, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732036579108 2024-11-19T17:16:23,132 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 61f399c154bb4ec4a8d7a7b07ed5577f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732036579193 2024-11-19T17:16:23,132 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 338e48e9869f40b98904f04e3dd1a1bf, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732036579193 2024-11-19T17:16:23,132 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b00b513e788464cb53fdd872398fdd7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732036581367 2024-11-19T17:16:23,132 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea71352aba58407496d60804b06d8385, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732036581366 2024-11-19T17:16:23,137 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:23,137 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6bc673cb3ab3290d1b783693a45a9bb#B#compaction#550 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:16:23,138 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/afc1ad7e24fe47a4a5478accad7a85b2 is 50, key is test_row_0/B:col10/1732036581367/Put/seqid=0 2024-11-19T17:16:23,138 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411192184db835b1a4660858bf557cc3d24d9_a6bc673cb3ab3290d1b783693a45a9bb store=[table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:23,140 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411192184db835b1a4660858bf557cc3d24d9_a6bc673cb3ab3290d1b783693a45a9bb, store=[table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:23,140 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192184db835b1a4660858bf557cc3d24d9_a6bc673cb3ab3290d1b783693a45a9bb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:23,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742475_1651 (size=12779) 2024-11-19T17:16:23,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742476_1652 (size=4469) 2024-11-19T17:16:23,145 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6bc673cb3ab3290d1b783693a45a9bb#A#compaction#551 average throughput is 3.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:16:23,146 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/bdca4cb631004cd1874f865afdb1fe2c is 175, key is test_row_0/A:col10/1732036581367/Put/seqid=0 2024-11-19T17:16:23,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742477_1653 (size=31840) 2024-11-19T17:16:23,153 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/bdca4cb631004cd1874f865afdb1fe2c as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/bdca4cb631004cd1874f865afdb1fe2c 2024-11-19T17:16:23,156 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6bc673cb3ab3290d1b783693a45a9bb/A of a6bc673cb3ab3290d1b783693a45a9bb into bdca4cb631004cd1874f865afdb1fe2c(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:16:23,156 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:23,156 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., storeName=a6bc673cb3ab3290d1b783693a45a9bb/A, priority=13, startTime=1732036583130; duration=0sec 2024-11-19T17:16:23,156 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:16:23,156 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6bc673cb3ab3290d1b783693a45a9bb:A 2024-11-19T17:16:23,157 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:16:23,157 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34835 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:16:23,157 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): a6bc673cb3ab3290d1b783693a45a9bb/C is initiating minor compaction (all files) 2024-11-19T17:16:23,157 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6bc673cb3ab3290d1b783693a45a9bb/C in TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:23,157 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/e6393ae902cb419aacb06bbd123f74a7, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/384a775e68074ec68ce5ccf941754475, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/da8e30ae7e3e43ba8fd0eb127f8a8683] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp, totalSize=34.0 K 2024-11-19T17:16:23,158 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6393ae902cb419aacb06bbd123f74a7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732036579108 2024-11-19T17:16:23,158 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 384a775e68074ec68ce5ccf941754475, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732036579193 2024-11-19T17:16:23,158 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting da8e30ae7e3e43ba8fd0eb127f8a8683, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732036581367 2024-11-19T17:16:23,163 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6bc673cb3ab3290d1b783693a45a9bb#C#compaction#552 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:16:23,164 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/a5786c556d7a4c40b3dfc442799a675c is 50, key is test_row_0/C:col10/1732036581367/Put/seqid=0 2024-11-19T17:16:23,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742478_1654 (size=12779) 2024-11-19T17:16:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T17:16:23,254 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:16:23,254 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:16:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees 2024-11-19T17:16:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-19T17:16:23,256 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:16:23,257 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:16:23,257 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:16:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-19T17:16:23,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-19T17:16:23,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:23,409 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2902): Flushing a6bc673cb3ab3290d1b783693a45a9bb 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-19T17:16:23,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=A 2024-11-19T17:16:23,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:16:23,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=B 2024-11-19T17:16:23,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:16:23,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=C 2024-11-19T17:16:23,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:16:23,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f3062c73db2943d1a7a57ace57e27d93_a6bc673cb3ab3290d1b783693a45a9bb is 50, key is test_row_0/A:col10/1732036581558/Put/seqid=0 2024-11-19T17:16:23,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742479_1655 (size=12454) 2024-11-19T17:16:23,548 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/afc1ad7e24fe47a4a5478accad7a85b2 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/afc1ad7e24fe47a4a5478accad7a85b2 2024-11-19T17:16:23,554 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6bc673cb3ab3290d1b783693a45a9bb/B of a6bc673cb3ab3290d1b783693a45a9bb into afc1ad7e24fe47a4a5478accad7a85b2(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:16:23,554 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:23,554 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., storeName=a6bc673cb3ab3290d1b783693a45a9bb/B, priority=13, startTime=1732036583130; duration=0sec 2024-11-19T17:16:23,554 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:23,554 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6bc673cb3ab3290d1b783693a45a9bb:B 2024-11-19T17:16:23,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-19T17:16:23,576 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/a5786c556d7a4c40b3dfc442799a675c as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/a5786c556d7a4c40b3dfc442799a675c 2024-11-19T17:16:23,580 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6bc673cb3ab3290d1b783693a45a9bb/C of a6bc673cb3ab3290d1b783693a45a9bb into a5786c556d7a4c40b3dfc442799a675c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:16:23,580 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:23,580 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., storeName=a6bc673cb3ab3290d1b783693a45a9bb/C, priority=13, startTime=1732036583131; duration=0sec 2024-11-19T17:16:23,580 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:23,580 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6bc673cb3ab3290d1b783693a45a9bb:C 2024-11-19T17:16:23,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:23,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:23,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036643703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:23,710 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_1' in a6bc673cb3ab3290d1b783693a45a9bb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:16:23,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:23,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:23,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:23,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036643710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:23,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036643714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:23,716 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_0' in a6bc673cb3ab3290d1b783693a45a9bb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:16:23,717 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:23,717 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:23,717 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:23,717 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:23,717 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:23,717 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:23,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:23,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:23,825 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f3062c73db2943d1a7a57ace57e27d93_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f3062c73db2943d1a7a57ace57e27d93_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036643824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:23,826 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:23,826 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:23,826 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:23,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/f4441f8472ad49929188cceabd2b1d52, store: [table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:23,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/f4441f8472ad49929188cceabd2b1d52 is 175, key is test_row_0/A:col10/1732036581558/Put/seqid=0 2024-11-19T17:16:23,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742480_1656 (size=31255) 2024-11-19T17:16:23,831 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=384, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/f4441f8472ad49929188cceabd2b1d52 2024-11-19T17:16:23,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/7318be4c87334c33b9e058c122213d16 is 50, key is test_row_0/B:col10/1732036581558/Put/seqid=0 2024-11-19T17:16:23,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742481_1657 (size=12301) 2024-11-19T17:16:23,855 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/7318be4c87334c33b9e058c122213d16 2024-11-19T17:16:23,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/4dcdd2e21152497080f0b548cc1012e7 is 50, key is test_row_0/C:col10/1732036581558/Put/seqid=0 2024-11-19T17:16:23,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742482_1658 (size=12301) 2024-11-19T17:16:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-19T17:16:24,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34418 deadline: 1732036644004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:24,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34398 deadline: 1732036644004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:24,009 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_1' in a6bc673cb3ab3290d1b783693a45a9bb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:16:24,009 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_1' in a6bc673cb3ab3290d1b783693a45a9bb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:16:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:24,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036644034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:24,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:24,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:24,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:24,265 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/4dcdd2e21152497080f0b548cc1012e7 2024-11-19T17:16:24,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/f4441f8472ad49929188cceabd2b1d52 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/f4441f8472ad49929188cceabd2b1d52 2024-11-19T17:16:24,272 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/f4441f8472ad49929188cceabd2b1d52, entries=150, sequenceid=384, filesize=30.5 K 2024-11-19T17:16:24,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/7318be4c87334c33b9e058c122213d16 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/7318be4c87334c33b9e058c122213d16 2024-11-19T17:16:24,279 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/7318be4c87334c33b9e058c122213d16, entries=150, sequenceid=384, filesize=12.0 K 2024-11-19T17:16:24,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/4dcdd2e21152497080f0b548cc1012e7 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/4dcdd2e21152497080f0b548cc1012e7 2024-11-19T17:16:24,284 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/4dcdd2e21152497080f0b548cc1012e7, entries=150, sequenceid=384, filesize=12.0 K 2024-11-19T17:16:24,285 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for a6bc673cb3ab3290d1b783693a45a9bb in 876ms, sequenceid=384, compaction requested=false 2024-11-19T17:16:24,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2603): Flush status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:24,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:24,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=173 2024-11-19T17:16:24,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=173 2024-11-19T17:16:24,289 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-19T17:16:24,289 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0290 sec 2024-11-19T17:16:24,292 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees in 1.0370 sec 2024-11-19T17:16:24,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:24,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6bc673cb3ab3290d1b783693a45a9bb 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T17:16:24,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=A 2024-11-19T17:16:24,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:24,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=B 2024-11-19T17:16:24,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:24,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=C 2024-11-19T17:16:24,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:24,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c0605575761a42efaab5c1f4e32b06a4_a6bc673cb3ab3290d1b783693a45a9bb is 50, key is test_row_0/A:col10/1732036583708/Put/seqid=0 2024-11-19T17:16:24,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742483_1659 (size=14994) 2024-11-19T17:16:24,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-19T17:16:24,394 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:16:24,395 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:16:24,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees 2024-11-19T17:16:24,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-19T17:16:24,396 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:16:24,397 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:16:24,397 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:16:24,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:24,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036644436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:24,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:24,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:24,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-19T17:16:24,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-19T17:16:24,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:24,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:24,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:24,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:24,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:24,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:24,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036644553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:24,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:24,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:24,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:24,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-19T17:16:24,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:24,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:24,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:24,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:24,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-19T17:16:24,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036644763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:24,766 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:24,766 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:24,766 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:24,776 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:24,779 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c0605575761a42efaab5c1f4e32b06a4_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c0605575761a42efaab5c1f4e32b06a4_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:24,780 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/673bd54f4a4d4166af4523034c49667b, store: [table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:24,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/673bd54f4a4d4166af4523034c49667b is 175, key is test_row_0/A:col10/1732036583708/Put/seqid=0 2024-11-19T17:16:24,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742484_1660 (size=39949) 2024-11-19T17:16:24,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-19T17:16:24,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:24,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:24,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:24,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:24,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:25,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-19T17:16:25,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:25,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:25,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:25,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:25,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:25,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:25,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-19T17:16:25,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:25,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036645074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:25,077 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:25,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:25,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:25,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-19T17:16:25,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:25,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:25,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:25,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:25,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:25,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:25,184 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=398, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/673bd54f4a4d4166af4523034c49667b 2024-11-19T17:16:25,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/81c439d7a051449d86c0ff2d658a9026 is 50, key is test_row_0/B:col10/1732036583708/Put/seqid=0 2024-11-19T17:16:25,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742485_1661 (size=12301) 2024-11-19T17:16:25,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-19T17:16:25,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:25,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:25,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:25,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:25,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:25,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:25,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-19T17:16:25,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:25,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:25,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:25,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:25,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:25,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-19T17:16:25,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036645583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:25,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/81c439d7a051449d86c0ff2d658a9026 2024-11-19T17:16:25,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/02cc0262c5c342ffb18f52da99862c48 is 50, key is test_row_0/C:col10/1732036583708/Put/seqid=0 2024-11-19T17:16:25,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742486_1662 (size=12301) 2024-11-19T17:16:25,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-19T17:16:25,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:25,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:25,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:25,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:25,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:25,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:25,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:25,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036645734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:25,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:25,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036645734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:25,738 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_1' in a6bc673cb3ab3290d1b783693a45a9bb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:16:25,738 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_0' in a6bc673cb3ab3290d1b783693a45a9bb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:16:25,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:25,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:25,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:25,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:25,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:25,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:25,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-19T17:16:25,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:25,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:25,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:25,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:25,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:25,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:25,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-19T17:16:25,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:25,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:25,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:25,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:25,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:25,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:26,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/02cc0262c5c342ffb18f52da99862c48 2024-11-19T17:16:26,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/673bd54f4a4d4166af4523034c49667b as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/673bd54f4a4d4166af4523034c49667b 2024-11-19T17:16:26,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/673bd54f4a4d4166af4523034c49667b, entries=200, sequenceid=398, filesize=39.0 K 2024-11-19T17:16:26,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/81c439d7a051449d86c0ff2d658a9026 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/81c439d7a051449d86c0ff2d658a9026 2024-11-19T17:16:26,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/81c439d7a051449d86c0ff2d658a9026, entries=150, sequenceid=398, filesize=12.0 K 2024-11-19T17:16:26,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/02cc0262c5c342ffb18f52da99862c48 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/02cc0262c5c342ffb18f52da99862c48 2024-11-19T17:16:26,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/02cc0262c5c342ffb18f52da99862c48, entries=150, sequenceid=398, filesize=12.0 K 2024-11-19T17:16:26,018 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a6bc673cb3ab3290d1b783693a45a9bb in 1671ms, sequenceid=398, compaction requested=true 2024-11-19T17:16:26,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:26,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6bc673cb3ab3290d1b783693a45a9bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:16:26,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:26,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6bc673cb3ab3290d1b783693a45a9bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:16:26,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:26,018 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:16:26,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6bc673cb3ab3290d1b783693a45a9bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:16:26,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:16:26,018 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:16:26,019 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103044 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:16:26,019 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): a6bc673cb3ab3290d1b783693a45a9bb/A is initiating minor compaction (all files) 2024-11-19T17:16:26,019 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6bc673cb3ab3290d1b783693a45a9bb/A in TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:26,019 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/bdca4cb631004cd1874f865afdb1fe2c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/f4441f8472ad49929188cceabd2b1d52, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/673bd54f4a4d4166af4523034c49667b] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp, totalSize=100.6 K 2024-11-19T17:16:26,019 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:26,019 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. files: [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/bdca4cb631004cd1874f865afdb1fe2c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/f4441f8472ad49929188cceabd2b1d52, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/673bd54f4a4d4166af4523034c49667b] 2024-11-19T17:16:26,020 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdca4cb631004cd1874f865afdb1fe2c, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732036579199 2024-11-19T17:16:26,020 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4441f8472ad49929188cceabd2b1d52, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732036581544 2024-11-19T17:16:26,020 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:16:26,020 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): a6bc673cb3ab3290d1b783693a45a9bb/B is initiating minor compaction (all files) 2024-11-19T17:16:26,020 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6bc673cb3ab3290d1b783693a45a9bb/B in TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:26,020 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/afc1ad7e24fe47a4a5478accad7a85b2, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/7318be4c87334c33b9e058c122213d16, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/81c439d7a051449d86c0ff2d658a9026] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp, totalSize=36.5 K 2024-11-19T17:16:26,021 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 673bd54f4a4d4166af4523034c49667b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732036583682 2024-11-19T17:16:26,021 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting afc1ad7e24fe47a4a5478accad7a85b2, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732036579199 2024-11-19T17:16:26,021 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 7318be4c87334c33b9e058c122213d16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732036581544 2024-11-19T17:16:26,021 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 81c439d7a051449d86c0ff2d658a9026, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732036583701 2024-11-19T17:16:26,025 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:26,026 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119b2fda30addfe4db18fa28e19ba299e53_a6bc673cb3ab3290d1b783693a45a9bb store=[table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:26,027 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6bc673cb3ab3290d1b783693a45a9bb#B#compaction#560 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:16:26,027 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/07c4bab95e524fe888ae3ac3b09be6d0 is 50, key is test_row_0/B:col10/1732036583708/Put/seqid=0 2024-11-19T17:16:26,028 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119b2fda30addfe4db18fa28e19ba299e53_a6bc673cb3ab3290d1b783693a45a9bb, store=[table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:26,028 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b2fda30addfe4db18fa28e19ba299e53_a6bc673cb3ab3290d1b783693a45a9bb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:26,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742487_1663 (size=12881) 2024-11-19T17:16:26,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742488_1664 (size=4469) 2024-11-19T17:16:26,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-19T17:16:26,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:26,079 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2902): Flushing a6bc673cb3ab3290d1b783693a45a9bb 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T17:16:26,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=A 2024-11-19T17:16:26,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:26,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=B 2024-11-19T17:16:26,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:26,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=C 2024-11-19T17:16:26,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:26,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119659e34c71fd34f329d40eac1531cd518_a6bc673cb3ab3290d1b783693a45a9bb is 50, key is test_row_0/A:col10/1732036584427/Put/seqid=0 2024-11-19T17:16:26,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742489_1665 (size=12454) 2024-11-19T17:16:26,437 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/07c4bab95e524fe888ae3ac3b09be6d0 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/07c4bab95e524fe888ae3ac3b09be6d0 2024-11-19T17:16:26,441 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6bc673cb3ab3290d1b783693a45a9bb#A#compaction#559 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:16:26,441 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/8450e5ec4e924df994edab8a7c299b87 is 175, key is test_row_0/A:col10/1732036583708/Put/seqid=0 2024-11-19T17:16:26,441 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6bc673cb3ab3290d1b783693a45a9bb/B of a6bc673cb3ab3290d1b783693a45a9bb into 07c4bab95e524fe888ae3ac3b09be6d0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:16:26,441 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:26,441 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., storeName=a6bc673cb3ab3290d1b783693a45a9bb/B, priority=13, startTime=1732036586018; duration=0sec 2024-11-19T17:16:26,441 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:16:26,441 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6bc673cb3ab3290d1b783693a45a9bb:B 2024-11-19T17:16:26,441 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:16:26,442 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:16:26,442 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): a6bc673cb3ab3290d1b783693a45a9bb/C is initiating minor compaction (all files) 2024-11-19T17:16:26,443 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6bc673cb3ab3290d1b783693a45a9bb/C in TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:26,443 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/a5786c556d7a4c40b3dfc442799a675c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/4dcdd2e21152497080f0b548cc1012e7, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/02cc0262c5c342ffb18f52da99862c48] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp, totalSize=36.5 K 2024-11-19T17:16:26,443 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting a5786c556d7a4c40b3dfc442799a675c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732036579199 2024-11-19T17:16:26,443 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dcdd2e21152497080f0b548cc1012e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732036581544 2024-11-19T17:16:26,444 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 02cc0262c5c342ffb18f52da99862c48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732036583701 2024-11-19T17:16:26,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742490_1666 (size=31835) 2024-11-19T17:16:26,449 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/8450e5ec4e924df994edab8a7c299b87 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/8450e5ec4e924df994edab8a7c299b87 2024-11-19T17:16:26,450 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6bc673cb3ab3290d1b783693a45a9bb#C#compaction#562 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:16:26,451 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/3806e399ca6e42d385d4b108d81e653d is 50, key is test_row_0/C:col10/1732036583708/Put/seqid=0 2024-11-19T17:16:26,454 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6bc673cb3ab3290d1b783693a45a9bb/A of a6bc673cb3ab3290d1b783693a45a9bb into 8450e5ec4e924df994edab8a7c299b87(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:16:26,454 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:26,454 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., storeName=a6bc673cb3ab3290d1b783693a45a9bb/A, priority=13, startTime=1732036586018; duration=0sec 2024-11-19T17:16:26,454 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:26,454 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6bc673cb3ab3290d1b783693a45a9bb:A 2024-11-19T17:16:26,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742491_1667 (size=12881) 2024-11-19T17:16:26,462 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/3806e399ca6e42d385d4b108d81e653d as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/3806e399ca6e42d385d4b108d81e653d 2024-11-19T17:16:26,465 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6bc673cb3ab3290d1b783693a45a9bb/C of a6bc673cb3ab3290d1b783693a45a9bb into 3806e399ca6e42d385d4b108d81e653d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:16:26,465 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:26,465 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., storeName=a6bc673cb3ab3290d1b783693a45a9bb/C, priority=13, startTime=1732036586018; duration=0sec 2024-11-19T17:16:26,465 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:26,466 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6bc673cb3ab3290d1b783693a45a9bb:C 2024-11-19T17:16:26,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:26,495 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119659e34c71fd34f329d40eac1531cd518_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119659e34c71fd34f329d40eac1531cd518_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:26,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/1705daec483e4439b5a096899a560d39, store: [table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:26,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/1705daec483e4439b5a096899a560d39 is 175, key is test_row_0/A:col10/1732036584427/Put/seqid=0 2024-11-19T17:16:26,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742492_1668 (size=31255) 2024-11-19T17:16:26,499 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=418, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/1705daec483e4439b5a096899a560d39 2024-11-19T17:16:26,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/7c0025ac5d0348148f94f2cbfec5962a is 50, key is test_row_0/B:col10/1732036584427/Put/seqid=0 2024-11-19T17:16:26,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742493_1669 (size=12301) 2024-11-19T17:16:26,510 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/7c0025ac5d0348148f94f2cbfec5962a 2024-11-19T17:16:26,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/920c898bfcfa4ac29b78a8c27f1704a2 is 50, key is test_row_0/C:col10/1732036584427/Put/seqid=0 2024-11-19T17:16:26,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742494_1670 (size=12301) 2024-11-19T17:16:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-19T17:16:26,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:26,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036646645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:26,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:26,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:26,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:26,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036646753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:26,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:26,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:26,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:26,920 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/920c898bfcfa4ac29b78a8c27f1704a2 2024-11-19T17:16:26,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/1705daec483e4439b5a096899a560d39 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/1705daec483e4439b5a096899a560d39 2024-11-19T17:16:26,926 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/1705daec483e4439b5a096899a560d39, entries=150, sequenceid=418, filesize=30.5 K 2024-11-19T17:16:26,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/7c0025ac5d0348148f94f2cbfec5962a as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/7c0025ac5d0348148f94f2cbfec5962a 2024-11-19T17:16:26,930 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/7c0025ac5d0348148f94f2cbfec5962a, entries=150, sequenceid=418, filesize=12.0 K 2024-11-19T17:16:26,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/920c898bfcfa4ac29b78a8c27f1704a2 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/920c898bfcfa4ac29b78a8c27f1704a2 2024-11-19T17:16:26,933 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/920c898bfcfa4ac29b78a8c27f1704a2, entries=150, sequenceid=418, filesize=12.0 K 2024-11-19T17:16:26,934 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a6bc673cb3ab3290d1b783693a45a9bb in 855ms, sequenceid=418, compaction requested=false 2024-11-19T17:16:26,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2603): Flush status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:26,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:26,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=175 2024-11-19T17:16:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=175 2024-11-19T17:16:26,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-19T17:16:26,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5380 sec 2024-11-19T17:16:26,939 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees in 2.5430 sec 2024-11-19T17:16:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:26,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6bc673cb3ab3290d1b783693a45a9bb 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T17:16:26,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=A 2024-11-19T17:16:26,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:26,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=B 2024-11-19T17:16:26,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:26,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=C 2024-11-19T17:16:26,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:26,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d1001dd414a4423bbf8fc4598261944b_a6bc673cb3ab3290d1b783693a45a9bb is 50, key is test_row_0/A:col10/1732036586964/Put/seqid=0 2024-11-19T17:16:26,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742495_1671 (size=14994) 2024-11-19T17:16:26,975 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:26,978 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d1001dd414a4423bbf8fc4598261944b_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d1001dd414a4423bbf8fc4598261944b_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:26,979 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/6e21de5b66f94a649b2d6ea5122e4ccb, store: [table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:26,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/6e21de5b66f94a649b2d6ea5122e4ccb is 175, key is test_row_0/A:col10/1732036586964/Put/seqid=0 2024-11-19T17:16:26,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742496_1672 (size=39949) 2024-11-19T17:16:27,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:27,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036647037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:27,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:27,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:27,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:27,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036647144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:27,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:27,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036647353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:27,357 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:27,357 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:27,357 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:27,383 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=434, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/6e21de5b66f94a649b2d6ea5122e4ccb 2024-11-19T17:16:27,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/db3e5e2bb22a44b98994c405bbb045b8 is 50, key is test_row_0/B:col10/1732036586964/Put/seqid=0 2024-11-19T17:16:27,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742497_1673 (size=12301) 2024-11-19T17:16:27,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036647664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:27,666 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:27,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/db3e5e2bb22a44b98994c405bbb045b8 2024-11-19T17:16:27,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/1ae6ef548acd4a28b1b1d3e13524c093 is 50, key is test_row_0/C:col10/1732036586964/Put/seqid=0 2024-11-19T17:16:27,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742498_1674 (size=12301) 2024-11-19T17:16:28,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:28,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036648173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:28,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:28,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:28,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:28,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/1ae6ef548acd4a28b1b1d3e13524c093 2024-11-19T17:16:28,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/6e21de5b66f94a649b2d6ea5122e4ccb as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/6e21de5b66f94a649b2d6ea5122e4ccb 2024-11-19T17:16:28,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/6e21de5b66f94a649b2d6ea5122e4ccb, entries=200, sequenceid=434, filesize=39.0 K 2024-11-19T17:16:28,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/db3e5e2bb22a44b98994c405bbb045b8 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/db3e5e2bb22a44b98994c405bbb045b8 2024-11-19T17:16:28,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/db3e5e2bb22a44b98994c405bbb045b8, entries=150, sequenceid=434, filesize=12.0 K 2024-11-19T17:16:28,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/1ae6ef548acd4a28b1b1d3e13524c093 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/1ae6ef548acd4a28b1b1d3e13524c093 2024-11-19T17:16:28,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/1ae6ef548acd4a28b1b1d3e13524c093, entries=150, sequenceid=434, filesize=12.0 K 2024-11-19T17:16:28,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a6bc673cb3ab3290d1b783693a45a9bb in 1252ms, sequenceid=434, compaction requested=true 2024-11-19T17:16:28,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:28,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6bc673cb3ab3290d1b783693a45a9bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:16:28,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:28,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6bc673cb3ab3290d1b783693a45a9bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:16:28,219 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:16:28,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:28,219 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:16:28,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6bc673cb3ab3290d1b783693a45a9bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:16:28,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:16:28,220 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:16:28,220 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:16:28,220 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): a6bc673cb3ab3290d1b783693a45a9bb/B is initiating minor compaction (all files) 2024-11-19T17:16:28,220 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): a6bc673cb3ab3290d1b783693a45a9bb/A is initiating minor compaction (all files) 2024-11-19T17:16:28,220 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6bc673cb3ab3290d1b783693a45a9bb/A in TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:28,220 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6bc673cb3ab3290d1b783693a45a9bb/B in TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:28,220 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/8450e5ec4e924df994edab8a7c299b87, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/1705daec483e4439b5a096899a560d39, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/6e21de5b66f94a649b2d6ea5122e4ccb] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp, totalSize=100.6 K 2024-11-19T17:16:28,220 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/07c4bab95e524fe888ae3ac3b09be6d0, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/7c0025ac5d0348148f94f2cbfec5962a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/db3e5e2bb22a44b98994c405bbb045b8] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp, totalSize=36.6 K 2024-11-19T17:16:28,220 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:28,220 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. files: [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/8450e5ec4e924df994edab8a7c299b87, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/1705daec483e4439b5a096899a560d39, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/6e21de5b66f94a649b2d6ea5122e4ccb] 2024-11-19T17:16:28,220 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 07c4bab95e524fe888ae3ac3b09be6d0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732036583701 2024-11-19T17:16:28,220 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8450e5ec4e924df994edab8a7c299b87, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732036583701 2024-11-19T17:16:28,221 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c0025ac5d0348148f94f2cbfec5962a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1732036584403 2024-11-19T17:16:28,221 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1705daec483e4439b5a096899a560d39, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1732036584403 2024-11-19T17:16:28,221 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting db3e5e2bb22a44b98994c405bbb045b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732036586625 2024-11-19T17:16:28,221 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e21de5b66f94a649b2d6ea5122e4ccb, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732036586625 2024-11-19T17:16:28,225 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:28,227 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6bc673cb3ab3290d1b783693a45a9bb#B#compaction#568 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:16:28,227 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411194d16f6a3db7440b1a902fd6d656d7bd4_a6bc673cb3ab3290d1b783693a45a9bb store=[table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:28,227 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/a209d513d4ae4f46ae8150196c8cb2ca is 50, key is test_row_0/B:col10/1732036586964/Put/seqid=0 2024-11-19T17:16:28,228 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411194d16f6a3db7440b1a902fd6d656d7bd4_a6bc673cb3ab3290d1b783693a45a9bb, store=[table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:28,228 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194d16f6a3db7440b1a902fd6d656d7bd4_a6bc673cb3ab3290d1b783693a45a9bb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:28,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742499_1675 (size=12983) 2024-11-19T17:16:28,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742500_1676 (size=4469) 2024-11-19T17:16:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-19T17:16:28,564 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:16:28,564 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:16:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees 2024-11-19T17:16:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-19T17:16:28,566 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:16:28,566 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=176, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:16:28,566 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=177, ppid=176, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:16:28,639 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/a209d513d4ae4f46ae8150196c8cb2ca as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/a209d513d4ae4f46ae8150196c8cb2ca 2024-11-19T17:16:28,639 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6bc673cb3ab3290d1b783693a45a9bb#A#compaction#569 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:16:28,639 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/c605e99d662d4e3a98b33c6b1ea6ef5c is 175, key is test_row_0/A:col10/1732036586964/Put/seqid=0 2024-11-19T17:16:28,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742501_1677 (size=31937) 2024-11-19T17:16:28,643 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6bc673cb3ab3290d1b783693a45a9bb/B of a6bc673cb3ab3290d1b783693a45a9bb into a209d513d4ae4f46ae8150196c8cb2ca(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:16:28,643 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:28,643 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., storeName=a6bc673cb3ab3290d1b783693a45a9bb/B, priority=13, startTime=1732036588219; duration=0sec 2024-11-19T17:16:28,643 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:16:28,643 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6bc673cb3ab3290d1b783693a45a9bb:B 2024-11-19T17:16:28,643 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:16:28,644 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:16:28,644 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): a6bc673cb3ab3290d1b783693a45a9bb/C is initiating minor compaction (all files) 2024-11-19T17:16:28,644 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6bc673cb3ab3290d1b783693a45a9bb/C in TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:28,644 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/3806e399ca6e42d385d4b108d81e653d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/920c898bfcfa4ac29b78a8c27f1704a2, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/1ae6ef548acd4a28b1b1d3e13524c093] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp, totalSize=36.6 K 2024-11-19T17:16:28,644 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 3806e399ca6e42d385d4b108d81e653d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732036583701 2024-11-19T17:16:28,645 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 920c898bfcfa4ac29b78a8c27f1704a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1732036584403 2024-11-19T17:16:28,645 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ae6ef548acd4a28b1b1d3e13524c093, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732036586625 2024-11-19T17:16:28,651 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6bc673cb3ab3290d1b783693a45a9bb#C#compaction#570 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:16:28,651 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/036314d5781842be9bb512b503177411 is 50, key is test_row_0/C:col10/1732036586964/Put/seqid=0 2024-11-19T17:16:28,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742502_1678 (size=12983) 2024-11-19T17:16:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-19T17:16:28,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-19T17:16:28,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:28,718 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2902): Flushing a6bc673cb3ab3290d1b783693a45a9bb 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T17:16:28,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=A 2024-11-19T17:16:28,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:28,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=B 2024-11-19T17:16:28,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:28,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=C 2024-11-19T17:16:28,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:28,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195793fcafed3740c5996d01dca44c13b5_a6bc673cb3ab3290d1b783693a45a9bb is 50, key is test_row_0/A:col10/1732036587017/Put/seqid=0 2024-11-19T17:16:28,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742503_1679 (size=12454) 2024-11-19T17:16:28,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:28,730 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195793fcafed3740c5996d01dca44c13b5_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195793fcafed3740c5996d01dca44c13b5_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:28,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/bf7e084b24cb46bc85bd3bd58a579d15, store: [table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:28,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/bf7e084b24cb46bc85bd3bd58a579d15 is 175, key is test_row_0/A:col10/1732036587017/Put/seqid=0 2024-11-19T17:16:28,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742504_1680 (size=31255) 2024-11-19T17:16:28,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-19T17:16:29,047 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/c605e99d662d4e3a98b33c6b1ea6ef5c as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/c605e99d662d4e3a98b33c6b1ea6ef5c 2024-11-19T17:16:29,051 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6bc673cb3ab3290d1b783693a45a9bb/A of a6bc673cb3ab3290d1b783693a45a9bb into c605e99d662d4e3a98b33c6b1ea6ef5c(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:16:29,051 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:29,051 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., storeName=a6bc673cb3ab3290d1b783693a45a9bb/A, priority=13, startTime=1732036588219; duration=0sec 2024-11-19T17:16:29,051 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:29,051 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6bc673cb3ab3290d1b783693a45a9bb:A 2024-11-19T17:16:29,059 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/036314d5781842be9bb512b503177411 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/036314d5781842be9bb512b503177411 2024-11-19T17:16:29,063 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6bc673cb3ab3290d1b783693a45a9bb/C of a6bc673cb3ab3290d1b783693a45a9bb into 036314d5781842be9bb512b503177411(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:16:29,063 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:29,063 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., storeName=a6bc673cb3ab3290d1b783693a45a9bb/C, priority=13, startTime=1732036588219; duration=0sec 2024-11-19T17:16:29,063 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:29,063 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6bc673cb3ab3290d1b783693a45a9bb:C 2024-11-19T17:16:29,135 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=454, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/bf7e084b24cb46bc85bd3bd58a579d15 2024-11-19T17:16:29,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/1d47e95b8d1a44dbab6443298a0c9447 is 50, key is test_row_0/B:col10/1732036587017/Put/seqid=0 2024-11-19T17:16:29,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742505_1681 (size=12301) 2024-11-19T17:16:29,156 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/1d47e95b8d1a44dbab6443298a0c9447 2024-11-19T17:16:29,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/4ef517cdc369402ab55c608c4ddb4089 is 50, key is test_row_0/C:col10/1732036587017/Put/seqid=0 2024-11-19T17:16:29,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742506_1682 (size=12301) 2024-11-19T17:16:29,165 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/4ef517cdc369402ab55c608c4ddb4089 2024-11-19T17:16:29,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/bf7e084b24cb46bc85bd3bd58a579d15 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/bf7e084b24cb46bc85bd3bd58a579d15 2024-11-19T17:16:29,172 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/bf7e084b24cb46bc85bd3bd58a579d15, entries=150, sequenceid=454, filesize=30.5 K 2024-11-19T17:16:29,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/1d47e95b8d1a44dbab6443298a0c9447 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/1d47e95b8d1a44dbab6443298a0c9447 2024-11-19T17:16:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,176 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/1d47e95b8d1a44dbab6443298a0c9447, entries=150, sequenceid=454, filesize=12.0 K 2024-11-19T17:16:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/4ef517cdc369402ab55c608c4ddb4089 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/4ef517cdc369402ab55c608c4ddb4089 2024-11-19T17:16:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,180 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/4ef517cdc369402ab55c608c4ddb4089, entries=150, sequenceid=454, filesize=12.0 K 2024-11-19T17:16:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,181 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for a6bc673cb3ab3290d1b783693a45a9bb in 463ms, sequenceid=454, compaction requested=false 2024-11-19T17:16:29,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2603): Flush status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=177 2024-11-19T17:16:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=177 2024-11-19T17:16:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=177, resume processing ppid=176 2024-11-19T17:16:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=177, ppid=176, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 616 msec 2024-11-19T17:16:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees in 621 msec 2024-11-19T17:16:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-19T17:16:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,194 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:16:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,194 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T17:16:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=178, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees 2024-11-19T17:16:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-19T17:16:29,195 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=178, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T17:16:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,196 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=178, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T17:16:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,196 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=179, ppid=178, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T17:16:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:29,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6bc673cb3ab3290d1b783693a45a9bb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T17:16:29,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=A 2024-11-19T17:16:29,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:29,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=B 2024-11-19T17:16:29,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:29,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=C 2024-11-19T17:16:29,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:29,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190827a01bbcd44522bbb380ffdb7968f5_a6bc673cb3ab3290d1b783693a45a9bb is 50, key is test_row_0/A:col10/1732036589244/Put/seqid=0 2024-11-19T17:16:29,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742507_1683 (size=14994) 2024-11-19T17:16:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-19T17:16:29,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036589326, max timestamp=1732036589342Num uniques 859; 2024-11-19T17:16:29,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036589326, max timestamp=1732036589342Num uniques 859; 2024-11-19T17:16:29,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732036589326, max timestamp=1732036589342Num uniques 859; 2024-11-19T17:16:29,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:16:29,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:16:29,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/280a42d09ec8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T17:16:29,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-19T17:16:29,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:29,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:29,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:29,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:29,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:29,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036649441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:29,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:29,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:29,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:29,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-19T17:16:29,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:29,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:29,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:29,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:29,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-19T17:16:29,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:29,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036649554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:29,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:29,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:29,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:29,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-19T17:16:29,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:29,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:29,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:29,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:29,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:29,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:29,667 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:29,670 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190827a01bbcd44522bbb380ffdb7968f5_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190827a01bbcd44522bbb380ffdb7968f5_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:29,670 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/eb07371e413848e2ae62325c72f76a1a, store: [table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:29,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/eb07371e413848e2ae62325c72f76a1a is 175, key is test_row_0/A:col10/1732036589244/Put/seqid=0 2024-11-19T17:16:29,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742508_1684 (size=39949) 2024-11-19T17:16:29,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:29,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34402 deadline: 1732036649744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:29,748 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_1' in a6bc673cb3ab3290d1b783693a45a9bb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:16:29,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:29,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:29,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:29,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:29,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34480 deadline: 1732036649764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:29,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:29,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036649764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:29,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:29,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:29,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:29,768 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 280a42d09ec8,33873,1732036422473 for 'test_row_0' in a6bc673cb3ab3290d1b783693a45a9bb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T17:16:29,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:29,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:29,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:29,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-19T17:16:29,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:29,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:29,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:29,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:29,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:29,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:29,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-19T17:16:29,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-19T17:16:29,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:29,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:29,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:29,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:29,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:29,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:30,075 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=466, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/eb07371e413848e2ae62325c72f76a1a 2024-11-19T17:16:30,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T17:16:30,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33873 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34492 deadline: 1732036650073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:30,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 , the old value is region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:30,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T17:16:30,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., hostname=280a42d09ec8,33873,1732036422473, seqNum=5 because the exception is null or not the one we care about 2024-11-19T17:16:30,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/3393810b6b28448291c545403695ac2c is 50, key is test_row_0/B:col10/1732036589244/Put/seqid=0 2024-11-19T17:16:30,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742509_1685 (size=12301) 2024-11-19T17:16:30,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/3393810b6b28448291c545403695ac2c 2024-11-19T17:16:30,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/9974874f28f041b5a615a3042aacbe1e is 50, key is test_row_0/C:col10/1732036589244/Put/seqid=0 2024-11-19T17:16:30,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742510_1686 (size=12301) 2024-11-19T17:16:30,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-19T17:16:30,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:30,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:30,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:30,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:30,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:30,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:30,173 INFO [Thread-2637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2637. 2024-11-19T17:16:30,173 DEBUG [Thread-2637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:16:30,173 DEBUG [Thread-2637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:30,173 DEBUG [Thread-2637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:30,173 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:16:30,174 INFO [Thread-2633 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2633. 2024-11-19T17:16:30,174 DEBUG [Thread-2633 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:16:30,174 DEBUG [Thread-2633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:30,175 DEBUG [Thread-2633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:30,175 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:16:30,175 INFO [Thread-2631 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2631. 2024-11-19T17:16:30,175 DEBUG [Thread-2631 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:16:30,176 DEBUG [Thread-2631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:30,176 DEBUG [Thread-2631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:30,176 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:16:30,177 INFO [Thread-2635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2635. 2024-11-19T17:16:30,177 DEBUG [Thread-2635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:16:30,177 DEBUG [Thread-2635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:30,177 DEBUG [Thread-2635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:30,177 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:16:30,178 INFO [Thread-2639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2639. 2024-11-19T17:16:30,178 DEBUG [Thread-2639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:16:30,178 DEBUG [Thread-2639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:30,178 DEBUG [Thread-2639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:30,178 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:16:30,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-19T17:16:30,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:30,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:30,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:30,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:30,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:30,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:30,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-19T17:16:30,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-19T17:16:30,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:30,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:30,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:30,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:30,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:30,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:16:30,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/9974874f28f041b5a615a3042aacbe1e 2024-11-19T17:16:30,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/eb07371e413848e2ae62325c72f76a1a as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/eb07371e413848e2ae62325c72f76a1a 2024-11-19T17:16:30,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/eb07371e413848e2ae62325c72f76a1a, entries=200, sequenceid=466, filesize=39.0 K 2024-11-19T17:16:30,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/3393810b6b28448291c545403695ac2c as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/3393810b6b28448291c545403695ac2c 2024-11-19T17:16:30,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/3393810b6b28448291c545403695ac2c, entries=150, sequenceid=466, filesize=12.0 K 2024-11-19T17:16:30,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/9974874f28f041b5a615a3042aacbe1e as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/9974874f28f041b5a615a3042aacbe1e 2024-11-19T17:16:30,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/9974874f28f041b5a615a3042aacbe1e, entries=150, sequenceid=466, filesize=12.0 K 2024-11-19T17:16:30,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a6bc673cb3ab3290d1b783693a45a9bb in 1248ms, sequenceid=466, compaction requested=true 2024-11-19T17:16:30,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:30,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6bc673cb3ab3290d1b783693a45a9bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T17:16:30,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:30,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6bc673cb3ab3290d1b783693a45a9bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T17:16:30,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:30,503 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:16:30,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6bc673cb3ab3290d1b783693a45a9bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T17:16:30,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:16:30,504 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:16:30,504 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:16:30,504 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:16:30,504 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1541): a6bc673cb3ab3290d1b783693a45a9bb/A is initiating minor compaction (all files) 2024-11-19T17:16:30,504 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): a6bc673cb3ab3290d1b783693a45a9bb/B is initiating minor compaction (all files) 2024-11-19T17:16:30,504 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6bc673cb3ab3290d1b783693a45a9bb/A in TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:30,504 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6bc673cb3ab3290d1b783693a45a9bb/B in TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:30,504 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/c605e99d662d4e3a98b33c6b1ea6ef5c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/bf7e084b24cb46bc85bd3bd58a579d15, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/eb07371e413848e2ae62325c72f76a1a] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp, totalSize=100.7 K 2024-11-19T17:16:30,504 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:30,504 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/a209d513d4ae4f46ae8150196c8cb2ca, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/1d47e95b8d1a44dbab6443298a0c9447, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/3393810b6b28448291c545403695ac2c] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp, totalSize=36.7 K 2024-11-19T17:16:30,504 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. files: [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/c605e99d662d4e3a98b33c6b1ea6ef5c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/bf7e084b24cb46bc85bd3bd58a579d15, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/eb07371e413848e2ae62325c72f76a1a] 2024-11-19T17:16:30,505 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting a209d513d4ae4f46ae8150196c8cb2ca, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732036586625 2024-11-19T17:16:30,505 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting c605e99d662d4e3a98b33c6b1ea6ef5c, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732036586625 2024-11-19T17:16:30,505 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d47e95b8d1a44dbab6443298a0c9447, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1732036587017 2024-11-19T17:16:30,505 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf7e084b24cb46bc85bd3bd58a579d15, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1732036587017 2024-11-19T17:16:30,505 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 3393810b6b28448291c545403695ac2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1732036589244 2024-11-19T17:16:30,505 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb07371e413848e2ae62325c72f76a1a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1732036589244 2024-11-19T17:16:30,509 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:30,510 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6bc673cb3ab3290d1b783693a45a9bb#B#compaction#577 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:16:30,510 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/49df549a3aba4d97ae424192c1800bdd is 50, key is test_row_0/B:col10/1732036589244/Put/seqid=0 2024-11-19T17:16:30,510 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411194b101094f32245a8a13fd089016f5118_a6bc673cb3ab3290d1b783693a45a9bb store=[table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:30,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742511_1687 (size=13085) 2024-11-19T17:16:30,515 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411194b101094f32245a8a13fd089016f5118_a6bc673cb3ab3290d1b783693a45a9bb, store=[table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:30,515 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194b101094f32245a8a13fd089016f5118_a6bc673cb3ab3290d1b783693a45a9bb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:30,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742512_1688 (size=4469) 2024-11-19T17:16:30,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-19T17:16:30,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:30,571 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2902): Flushing a6bc673cb3ab3290d1b783693a45a9bb 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-19T17:16:30,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=A 2024-11-19T17:16:30,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:16:30,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=B 2024-11-19T17:16:30,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:16:30,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=C 2024-11-19T17:16:30,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T17:16:30,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111901ca7300b8de44a3adbce7e5d51ac236_a6bc673cb3ab3290d1b783693a45a9bb is 50, key is test_row_0/A:col10/1732036589380/Put/seqid=0 2024-11-19T17:16:30,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742513_1689 (size=12454) 2024-11-19T17:16:30,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33873 {}] regionserver.HRegion(8855): Flush requested on a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:30,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. as already flushing 2024-11-19T17:16:30,585 INFO [Thread-2622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2622. 2024-11-19T17:16:30,585 DEBUG [Thread-2622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:16:30,585 DEBUG [Thread-2622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:30,585 DEBUG [Thread-2622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:30,585 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:16:30,917 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/49df549a3aba4d97ae424192c1800bdd as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/49df549a3aba4d97ae424192c1800bdd 2024-11-19T17:16:30,920 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6bc673cb3ab3290d1b783693a45a9bb/B of a6bc673cb3ab3290d1b783693a45a9bb into 49df549a3aba4d97ae424192c1800bdd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:16:30,920 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:30,920 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., storeName=a6bc673cb3ab3290d1b783693a45a9bb/B, priority=13, startTime=1732036590503; duration=0sec 2024-11-19T17:16:30,920 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T17:16:30,920 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6bc673cb3ab3290d1b783693a45a9bb:B 2024-11-19T17:16:30,920 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T17:16:30,921 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6bc673cb3ab3290d1b783693a45a9bb#A#compaction#578 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:16:30,921 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T17:16:30,921 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1541): a6bc673cb3ab3290d1b783693a45a9bb/C is initiating minor compaction (all files) 2024-11-19T17:16:30,921 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6bc673cb3ab3290d1b783693a45a9bb/C in TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:30,921 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/036314d5781842be9bb512b503177411, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/4ef517cdc369402ab55c608c4ddb4089, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/9974874f28f041b5a615a3042aacbe1e] into tmpdir=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp, totalSize=36.7 K 2024-11-19T17:16:30,921 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 036314d5781842be9bb512b503177411, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732036586625 2024-11-19T17:16:30,921 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/641144a42ddf4fd29df1e2d19929f252 is 175, key is test_row_0/A:col10/1732036589244/Put/seqid=0 2024-11-19T17:16:30,921 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ef517cdc369402ab55c608c4ddb4089, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1732036587017 2024-11-19T17:16:30,922 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] compactions.Compactor(225): Compacting 9974874f28f041b5a615a3042aacbe1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1732036589244 2024-11-19T17:16:30,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742514_1690 (size=32039) 2024-11-19T17:16:30,927 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/641144a42ddf4fd29df1e2d19929f252 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/641144a42ddf4fd29df1e2d19929f252 2024-11-19T17:16:30,927 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6bc673cb3ab3290d1b783693a45a9bb#C#compaction#580 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T17:16:30,927 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/7d21df0a894e4a10b5409045703912de is 50, key is test_row_0/C:col10/1732036589244/Put/seqid=0 2024-11-19T17:16:30,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742515_1691 (size=13085) 2024-11-19T17:16:30,930 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6bc673cb3ab3290d1b783693a45a9bb/A of a6bc673cb3ab3290d1b783693a45a9bb into 641144a42ddf4fd29df1e2d19929f252(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:16:30,930 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:30,930 INFO [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., storeName=a6bc673cb3ab3290d1b783693a45a9bb/A, priority=13, startTime=1732036590503; duration=0sec 2024-11-19T17:16:30,930 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:30,930 DEBUG [RS:0;280a42d09ec8:33873-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6bc673cb3ab3290d1b783693a45a9bb:A 2024-11-19T17:16:30,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:30,982 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111901ca7300b8de44a3adbce7e5d51ac236_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111901ca7300b8de44a3adbce7e5d51ac236_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:30,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/90e1ee873c624f69a8ad1e0d923b68ff, store: [table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:30,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/90e1ee873c624f69a8ad1e0d923b68ff is 175, key is test_row_0/A:col10/1732036589380/Put/seqid=0 2024-11-19T17:16:30,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742516_1692 (size=31255) 2024-11-19T17:16:31,333 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/7d21df0a894e4a10b5409045703912de as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/7d21df0a894e4a10b5409045703912de 2024-11-19T17:16:31,336 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6bc673cb3ab3290d1b783693a45a9bb/C of a6bc673cb3ab3290d1b783693a45a9bb into 7d21df0a894e4a10b5409045703912de(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T17:16:31,336 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:31,336 INFO [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., storeName=a6bc673cb3ab3290d1b783693a45a9bb/C, priority=13, startTime=1732036590503; duration=0sec 2024-11-19T17:16:31,336 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T17:16:31,336 DEBUG [RS:0;280a42d09ec8:33873-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6bc673cb3ab3290d1b783693a45a9bb:C 2024-11-19T17:16:31,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-19T17:16:31,386 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=503, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/90e1ee873c624f69a8ad1e0d923b68ff 2024-11-19T17:16:31,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/ae19d5cb9e4a4e8598ad96b32f1158ed is 50, key is test_row_0/B:col10/1732036589380/Put/seqid=0 2024-11-19T17:16:31,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742517_1693 (size=12301) 2024-11-19T17:16:31,795 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/ae19d5cb9e4a4e8598ad96b32f1158ed 2024-11-19T17:16:31,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/67c72acf8d37444c9d2d30c4b7e70a4f is 50, key is test_row_0/C:col10/1732036589380/Put/seqid=0 2024-11-19T17:16:31,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742518_1694 (size=12301) 2024-11-19T17:16:32,203 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/67c72acf8d37444c9d2d30c4b7e70a4f 2024-11-19T17:16:32,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/90e1ee873c624f69a8ad1e0d923b68ff as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/90e1ee873c624f69a8ad1e0d923b68ff 2024-11-19T17:16:32,209 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/90e1ee873c624f69a8ad1e0d923b68ff, entries=150, sequenceid=503, filesize=30.5 K 2024-11-19T17:16:32,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/ae19d5cb9e4a4e8598ad96b32f1158ed as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/ae19d5cb9e4a4e8598ad96b32f1158ed 2024-11-19T17:16:32,211 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/ae19d5cb9e4a4e8598ad96b32f1158ed, entries=150, sequenceid=503, filesize=12.0 K 2024-11-19T17:16:32,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/67c72acf8d37444c9d2d30c4b7e70a4f as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/67c72acf8d37444c9d2d30c4b7e70a4f 2024-11-19T17:16:32,214 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/67c72acf8d37444c9d2d30c4b7e70a4f, entries=150, sequenceid=503, filesize=12.0 K 2024-11-19T17:16:32,214 INFO [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=6.71 KB/6870 for a6bc673cb3ab3290d1b783693a45a9bb in 1643ms, sequenceid=503, compaction requested=false 2024-11-19T17:16:32,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2603): Flush status journal for a6bc673cb3ab3290d1b783693a45a9bb: 2024-11-19T17:16:32,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:32,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/280a42d09ec8:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=179 2024-11-19T17:16:32,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster(4169): Remote procedure done, pid=179 2024-11-19T17:16:32,217 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=179, resume processing ppid=178 2024-11-19T17:16:32,217 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=179, ppid=178, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0200 sec 2024-11-19T17:16:32,218 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=178, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees in 3.0230 sec 2024-11-19T17:16:33,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-19T17:16:33,364 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:16:34,095 INFO [Thread-2628 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2628. 2024-11-19T17:16:34,095 DEBUG [Thread-2628 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:16:34,095 INFO [Thread-2624 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2624. 2024-11-19T17:16:34,095 DEBUG [Thread-2628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:34,095 DEBUG [Thread-2624 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:16:34,095 DEBUG [Thread-2624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:34,095 DEBUG [Thread-2628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:34,095 DEBUG [Thread-2624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:34,095 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:16:34,096 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:16:39,815 INFO [Thread-2626 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2626. 2024-11-19T17:16:39,815 DEBUG [Thread-2626 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:16:39,815 DEBUG [Thread-2626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:39,815 DEBUG [Thread-2626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:39,816 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:16:39,844 INFO [Thread-2620 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2620. 2024-11-19T17:16:39,844 DEBUG [Thread-2620 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T17:16:39,844 DEBUG [Thread-2620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:39,845 DEBUG [Thread-2620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:39,845 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:16:39,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T17:16:39,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-19T17:16:39,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 191 2024-11-19T17:16:39,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-19T17:16:39,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-19T17:16:39,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-19T17:16:39,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T17:16:39,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2793 2024-11-19T17:16:39,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2816 2024-11-19T17:16:39,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2768 2024-11-19T17:16:39,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2805 2024-11-19T17:16:39,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2837 2024-11-19T17:16:39,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T17:16:39,845 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T17:16:39,845 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T17:16:39,845 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:39,845 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:39,845 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:16:39,846 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T17:16:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=180, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T17:16:39,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-19T17:16:39,848 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732036599848"}]},"ts":"1732036599848"} 2024-11-19T17:16:39,849 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T17:16:39,849 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T17:16:39,850 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=181, ppid=180, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T17:16:39,851 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=182, ppid=181, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6bc673cb3ab3290d1b783693a45a9bb, UNASSIGN}] 2024-11-19T17:16:39,851 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=182, ppid=181, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6bc673cb3ab3290d1b783693a45a9bb, UNASSIGN 2024-11-19T17:16:39,852 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=182 updating hbase:meta row=a6bc673cb3ab3290d1b783693a45a9bb, regionState=CLOSING, regionLocation=280a42d09ec8,33873,1732036422473 2024-11-19T17:16:39,853 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=182, ppid=181, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6bc673cb3ab3290d1b783693a45a9bb, UNASSIGN because future has completed 2024-11-19T17:16:39,853 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T17:16:39,853 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=183, ppid=182, state=RUNNABLE, hasLock=false; CloseRegionProcedure a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473}] 2024-11-19T17:16:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-19T17:16:40,005 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] handler.UnassignRegionHandler(122): Close a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:40,005 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T17:16:40,005 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegion(1722): Closing a6bc673cb3ab3290d1b783693a45a9bb, disabling compactions & flushes 2024-11-19T17:16:40,005 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:40,005 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:40,005 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. after waiting 0 ms 2024-11-19T17:16:40,005 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:40,005 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegion(2902): Flushing a6bc673cb3ab3290d1b783693a45a9bb 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T17:16:40,006 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=A 2024-11-19T17:16:40,006 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:40,006 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=B 2024-11-19T17:16:40,006 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:40,006 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6bc673cb3ab3290d1b783693a45a9bb, store=C 2024-11-19T17:16:40,006 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T17:16:40,010 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e602718d14be489e8de3d9b9edfeb3d7_a6bc673cb3ab3290d1b783693a45a9bb is 50, key is test_row_0/A:col10/1732036599814/Put/seqid=0 2024-11-19T17:16:40,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742519_1695 (size=9914) 2024-11-19T17:16:40,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-19T17:16:40,414 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:16:40,417 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e602718d14be489e8de3d9b9edfeb3d7_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e602718d14be489e8de3d9b9edfeb3d7_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:40,417 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/7a803bf4ca6047dda56f37a1518dd218, store: [table=TestAcidGuarantees family=A region=a6bc673cb3ab3290d1b783693a45a9bb] 2024-11-19T17:16:40,418 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/7a803bf4ca6047dda56f37a1518dd218 is 175, key is test_row_0/A:col10/1732036599814/Put/seqid=0 2024-11-19T17:16:40,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742520_1696 (size=22561) 2024-11-19T17:16:40,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-19T17:16:40,710 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T17:16:40,821 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=514, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/7a803bf4ca6047dda56f37a1518dd218 2024-11-19T17:16:40,826 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/e96584f6b9234158905d9dc22729c926 is 50, key is test_row_0/B:col10/1732036599814/Put/seqid=0 2024-11-19T17:16:40,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742521_1697 (size=9857) 2024-11-19T17:16:40,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-19T17:16:41,229 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/e96584f6b9234158905d9dc22729c926 2024-11-19T17:16:41,234 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/b89ec53b51ca4a98b60e5418de53e889 is 50, key is test_row_0/C:col10/1732036599814/Put/seqid=0 2024-11-19T17:16:41,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742522_1698 (size=9857) 2024-11-19T17:16:41,638 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/b89ec53b51ca4a98b60e5418de53e889 2024-11-19T17:16:41,641 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/A/7a803bf4ca6047dda56f37a1518dd218 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/7a803bf4ca6047dda56f37a1518dd218 2024-11-19T17:16:41,644 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/7a803bf4ca6047dda56f37a1518dd218, entries=100, sequenceid=514, filesize=22.0 K 2024-11-19T17:16:41,644 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/B/e96584f6b9234158905d9dc22729c926 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/e96584f6b9234158905d9dc22729c926 2024-11-19T17:16:41,647 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/e96584f6b9234158905d9dc22729c926, entries=100, sequenceid=514, filesize=9.6 K 2024-11-19T17:16:41,647 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/.tmp/C/b89ec53b51ca4a98b60e5418de53e889 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/b89ec53b51ca4a98b60e5418de53e889 2024-11-19T17:16:41,649 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/b89ec53b51ca4a98b60e5418de53e889, entries=100, sequenceid=514, filesize=9.6 K 2024-11-19T17:16:41,650 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for a6bc673cb3ab3290d1b783693a45a9bb in 1645ms, sequenceid=514, compaction requested=true 2024-11-19T17:16:41,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/7d0505b848ff426dafddf8790c31383f, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/0e0a721fd23c4bbbab6a272bace38421, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/d00c358256c84117895aeeaa99ad5cef, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/69908ceb18234adf8e5d9dce96b21e35, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/2f7287f33446461cbd9ee08cd69180d9, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/1c6d923ffb1c4ce481f3da7650447edd, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/bfefa39f18db45b98aac0516d66b80f5, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/b31a419094e9403c920dec72cd297cd8, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/dea6954c8cd04918aa80a6a8803614bf, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/abd5ca30f7c247e0a6ed71ecb19b4c9d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/75af5866857444ea9ae2a4c677ffe516, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/a06751b29faa4f47b3d332e541d36f4c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/338e48e9869f40b98904f04e3dd1a1bf, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/bdca4cb631004cd1874f865afdb1fe2c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/ea71352aba58407496d60804b06d8385, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/f4441f8472ad49929188cceabd2b1d52, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/673bd54f4a4d4166af4523034c49667b, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/8450e5ec4e924df994edab8a7c299b87, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/1705daec483e4439b5a096899a560d39, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/6e21de5b66f94a649b2d6ea5122e4ccb, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/c605e99d662d4e3a98b33c6b1ea6ef5c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/bf7e084b24cb46bc85bd3bd58a579d15, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/eb07371e413848e2ae62325c72f76a1a] to archive 2024-11-19T17:16:41,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:16:41,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/7d0505b848ff426dafddf8790c31383f to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/7d0505b848ff426dafddf8790c31383f 2024-11-19T17:16:41,653 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/0e0a721fd23c4bbbab6a272bace38421 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/0e0a721fd23c4bbbab6a272bace38421 2024-11-19T17:16:41,654 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/d00c358256c84117895aeeaa99ad5cef to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/d00c358256c84117895aeeaa99ad5cef 2024-11-19T17:16:41,655 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/69908ceb18234adf8e5d9dce96b21e35 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/69908ceb18234adf8e5d9dce96b21e35 2024-11-19T17:16:41,655 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/2f7287f33446461cbd9ee08cd69180d9 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/2f7287f33446461cbd9ee08cd69180d9 2024-11-19T17:16:41,656 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/1c6d923ffb1c4ce481f3da7650447edd to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/1c6d923ffb1c4ce481f3da7650447edd 2024-11-19T17:16:41,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/bfefa39f18db45b98aac0516d66b80f5 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/bfefa39f18db45b98aac0516d66b80f5 2024-11-19T17:16:41,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/b31a419094e9403c920dec72cd297cd8 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/b31a419094e9403c920dec72cd297cd8 2024-11-19T17:16:41,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/dea6954c8cd04918aa80a6a8803614bf to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/dea6954c8cd04918aa80a6a8803614bf 2024-11-19T17:16:41,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/abd5ca30f7c247e0a6ed71ecb19b4c9d to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/abd5ca30f7c247e0a6ed71ecb19b4c9d 2024-11-19T17:16:41,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/75af5866857444ea9ae2a4c677ffe516 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/75af5866857444ea9ae2a4c677ffe516 2024-11-19T17:16:41,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/a06751b29faa4f47b3d332e541d36f4c to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/a06751b29faa4f47b3d332e541d36f4c 2024-11-19T17:16:41,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/338e48e9869f40b98904f04e3dd1a1bf to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/338e48e9869f40b98904f04e3dd1a1bf 2024-11-19T17:16:41,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/bdca4cb631004cd1874f865afdb1fe2c to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/bdca4cb631004cd1874f865afdb1fe2c 2024-11-19T17:16:41,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/ea71352aba58407496d60804b06d8385 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/ea71352aba58407496d60804b06d8385 2024-11-19T17:16:41,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/f4441f8472ad49929188cceabd2b1d52 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/f4441f8472ad49929188cceabd2b1d52 2024-11-19T17:16:41,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/673bd54f4a4d4166af4523034c49667b to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/673bd54f4a4d4166af4523034c49667b 2024-11-19T17:16:41,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/8450e5ec4e924df994edab8a7c299b87 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/8450e5ec4e924df994edab8a7c299b87 2024-11-19T17:16:41,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/1705daec483e4439b5a096899a560d39 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/1705daec483e4439b5a096899a560d39 2024-11-19T17:16:41,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/6e21de5b66f94a649b2d6ea5122e4ccb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/6e21de5b66f94a649b2d6ea5122e4ccb 2024-11-19T17:16:41,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/c605e99d662d4e3a98b33c6b1ea6ef5c to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/c605e99d662d4e3a98b33c6b1ea6ef5c 2024-11-19T17:16:41,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/bf7e084b24cb46bc85bd3bd58a579d15 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/bf7e084b24cb46bc85bd3bd58a579d15 2024-11-19T17:16:41,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/eb07371e413848e2ae62325c72f76a1a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/eb07371e413848e2ae62325c72f76a1a 2024-11-19T17:16:41,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/f637db8a86af408f97f22de974db9de4, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/8a01da8a613c4d84be127d8e68e44f46, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/2e9abe86edae40edb805d1a303cb9622, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/4199c00a41ce42dcab467923ee4cd761, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/ccfc5882261a4c7b9e67d9ba9e1300f0, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/8e5eff5b73fa4149a4cd3cb7723db4bc, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/6b12cf53a5964a55afe9e7572c64c1fc, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/e4b8fe5f444f46e4a8a3cbf07ea7d187, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/6889ecc63cd74493b4ed9a007e777cfb, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/a1c5e88a989142079c7cc937eda9e0ec, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/74c0ce18a2434242af71b062319b5cc0, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/6a04037055f04162b7571296360e3c1f, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/61f399c154bb4ec4a8d7a7b07ed5577f, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/afc1ad7e24fe47a4a5478accad7a85b2, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/5b00b513e788464cb53fdd872398fdd7, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/7318be4c87334c33b9e058c122213d16, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/07c4bab95e524fe888ae3ac3b09be6d0, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/81c439d7a051449d86c0ff2d658a9026, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/7c0025ac5d0348148f94f2cbfec5962a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/a209d513d4ae4f46ae8150196c8cb2ca, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/db3e5e2bb22a44b98994c405bbb045b8, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/1d47e95b8d1a44dbab6443298a0c9447, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/3393810b6b28448291c545403695ac2c] to archive 2024-11-19T17:16:41,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:16:41,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/f637db8a86af408f97f22de974db9de4 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/f637db8a86af408f97f22de974db9de4 2024-11-19T17:16:41,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/8a01da8a613c4d84be127d8e68e44f46 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/8a01da8a613c4d84be127d8e68e44f46 2024-11-19T17:16:41,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/2e9abe86edae40edb805d1a303cb9622 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/2e9abe86edae40edb805d1a303cb9622 2024-11-19T17:16:41,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/4199c00a41ce42dcab467923ee4cd761 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/4199c00a41ce42dcab467923ee4cd761 2024-11-19T17:16:41,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/ccfc5882261a4c7b9e67d9ba9e1300f0 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/ccfc5882261a4c7b9e67d9ba9e1300f0 2024-11-19T17:16:41,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/8e5eff5b73fa4149a4cd3cb7723db4bc to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/8e5eff5b73fa4149a4cd3cb7723db4bc 2024-11-19T17:16:41,676 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/6b12cf53a5964a55afe9e7572c64c1fc to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/6b12cf53a5964a55afe9e7572c64c1fc 2024-11-19T17:16:41,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/e4b8fe5f444f46e4a8a3cbf07ea7d187 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/e4b8fe5f444f46e4a8a3cbf07ea7d187 2024-11-19T17:16:41,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/6889ecc63cd74493b4ed9a007e777cfb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/6889ecc63cd74493b4ed9a007e777cfb 2024-11-19T17:16:41,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/a1c5e88a989142079c7cc937eda9e0ec to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/a1c5e88a989142079c7cc937eda9e0ec 2024-11-19T17:16:41,679 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/74c0ce18a2434242af71b062319b5cc0 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/74c0ce18a2434242af71b062319b5cc0 2024-11-19T17:16:41,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/6a04037055f04162b7571296360e3c1f to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/6a04037055f04162b7571296360e3c1f 2024-11-19T17:16:41,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/61f399c154bb4ec4a8d7a7b07ed5577f to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/61f399c154bb4ec4a8d7a7b07ed5577f 2024-11-19T17:16:41,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/afc1ad7e24fe47a4a5478accad7a85b2 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/afc1ad7e24fe47a4a5478accad7a85b2 2024-11-19T17:16:41,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/5b00b513e788464cb53fdd872398fdd7 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/5b00b513e788464cb53fdd872398fdd7 2024-11-19T17:16:41,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/7318be4c87334c33b9e058c122213d16 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/7318be4c87334c33b9e058c122213d16 2024-11-19T17:16:41,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/07c4bab95e524fe888ae3ac3b09be6d0 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/07c4bab95e524fe888ae3ac3b09be6d0 2024-11-19T17:16:41,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/81c439d7a051449d86c0ff2d658a9026 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/81c439d7a051449d86c0ff2d658a9026 2024-11-19T17:16:41,685 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/7c0025ac5d0348148f94f2cbfec5962a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/7c0025ac5d0348148f94f2cbfec5962a 2024-11-19T17:16:41,686 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/a209d513d4ae4f46ae8150196c8cb2ca to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/a209d513d4ae4f46ae8150196c8cb2ca 2024-11-19T17:16:41,686 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/db3e5e2bb22a44b98994c405bbb045b8 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/db3e5e2bb22a44b98994c405bbb045b8 2024-11-19T17:16:41,687 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/1d47e95b8d1a44dbab6443298a0c9447 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/1d47e95b8d1a44dbab6443298a0c9447 2024-11-19T17:16:41,688 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/3393810b6b28448291c545403695ac2c to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/3393810b6b28448291c545403695ac2c 2024-11-19T17:16:41,689 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/ac84d178d35d4a15a94bae66f890db1d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/d6e9b245ec364c89b91069eaec402c4a, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/a506325c528d46aa95bae864101bde45, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/0f98c9a4e64c4a9cabe51c8b12f6c346, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/fd389427fe1f40328f013ace7b84111c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/e246b35c30eb46b787c366447c0985a8, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/320ef2e2ee364c1ba19dc903edae35c8, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/ef242198db4a488ea84989408202ef67, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/adbb3e84c19e4ccaac318ae0d464f3c0, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/6f2228602b9343f4a574e85b39491889, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/e6393ae902cb419aacb06bbd123f74a7, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/e960304e5e794d1689a07d16fe7d76b1, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/384a775e68074ec68ce5ccf941754475, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/a5786c556d7a4c40b3dfc442799a675c, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/da8e30ae7e3e43ba8fd0eb127f8a8683, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/4dcdd2e21152497080f0b548cc1012e7, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/3806e399ca6e42d385d4b108d81e653d, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/02cc0262c5c342ffb18f52da99862c48, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/920c898bfcfa4ac29b78a8c27f1704a2, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/036314d5781842be9bb512b503177411, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/1ae6ef548acd4a28b1b1d3e13524c093, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/4ef517cdc369402ab55c608c4ddb4089, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/9974874f28f041b5a615a3042aacbe1e] to archive 2024-11-19T17:16:41,690 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T17:16:41,690 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/ac84d178d35d4a15a94bae66f890db1d to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/ac84d178d35d4a15a94bae66f890db1d 2024-11-19T17:16:41,691 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/d6e9b245ec364c89b91069eaec402c4a to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/d6e9b245ec364c89b91069eaec402c4a 2024-11-19T17:16:41,692 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/a506325c528d46aa95bae864101bde45 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/a506325c528d46aa95bae864101bde45 2024-11-19T17:16:41,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/0f98c9a4e64c4a9cabe51c8b12f6c346 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/0f98c9a4e64c4a9cabe51c8b12f6c346 2024-11-19T17:16:41,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/fd389427fe1f40328f013ace7b84111c to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/fd389427fe1f40328f013ace7b84111c 2024-11-19T17:16:41,694 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/e246b35c30eb46b787c366447c0985a8 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/e246b35c30eb46b787c366447c0985a8 2024-11-19T17:16:41,694 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/320ef2e2ee364c1ba19dc903edae35c8 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/320ef2e2ee364c1ba19dc903edae35c8 2024-11-19T17:16:41,695 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/ef242198db4a488ea84989408202ef67 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/ef242198db4a488ea84989408202ef67 2024-11-19T17:16:41,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/adbb3e84c19e4ccaac318ae0d464f3c0 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/adbb3e84c19e4ccaac318ae0d464f3c0 2024-11-19T17:16:41,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/6f2228602b9343f4a574e85b39491889 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/6f2228602b9343f4a574e85b39491889 2024-11-19T17:16:41,697 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/e6393ae902cb419aacb06bbd123f74a7 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/e6393ae902cb419aacb06bbd123f74a7 2024-11-19T17:16:41,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/e960304e5e794d1689a07d16fe7d76b1 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/e960304e5e794d1689a07d16fe7d76b1 2024-11-19T17:16:41,699 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/384a775e68074ec68ce5ccf941754475 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/384a775e68074ec68ce5ccf941754475 2024-11-19T17:16:41,699 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/a5786c556d7a4c40b3dfc442799a675c to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/a5786c556d7a4c40b3dfc442799a675c 2024-11-19T17:16:41,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/da8e30ae7e3e43ba8fd0eb127f8a8683 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/da8e30ae7e3e43ba8fd0eb127f8a8683 2024-11-19T17:16:41,701 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/4dcdd2e21152497080f0b548cc1012e7 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/4dcdd2e21152497080f0b548cc1012e7 2024-11-19T17:16:41,701 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/3806e399ca6e42d385d4b108d81e653d to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/3806e399ca6e42d385d4b108d81e653d 2024-11-19T17:16:41,702 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/02cc0262c5c342ffb18f52da99862c48 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/02cc0262c5c342ffb18f52da99862c48 2024-11-19T17:16:41,703 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/920c898bfcfa4ac29b78a8c27f1704a2 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/920c898bfcfa4ac29b78a8c27f1704a2 2024-11-19T17:16:41,703 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/036314d5781842be9bb512b503177411 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/036314d5781842be9bb512b503177411 2024-11-19T17:16:41,704 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/1ae6ef548acd4a28b1b1d3e13524c093 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/1ae6ef548acd4a28b1b1d3e13524c093 2024-11-19T17:16:41,705 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/4ef517cdc369402ab55c608c4ddb4089 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/4ef517cdc369402ab55c608c4ddb4089 2024-11-19T17:16:41,705 DEBUG [StoreCloser-TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/9974874f28f041b5a615a3042aacbe1e to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/9974874f28f041b5a615a3042aacbe1e 2024-11-19T17:16:41,709 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/recovered.edits/517.seqid, newMaxSeqId=517, maxSeqId=4 2024-11-19T17:16:41,709 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. 2024-11-19T17:16:41,709 DEBUG [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegion(1676): Region close journal for a6bc673cb3ab3290d1b783693a45a9bb: Waiting for close lock at 1732036600005Running coprocessor pre-close hooks at 1732036600005Disabling compacts and flushes for region at 1732036600005Disabling writes for close at 1732036600005Obtaining lock to block concurrent updates at 1732036600006 (+1 ms)Preparing flush snapshotting stores in a6bc673cb3ab3290d1b783693a45a9bb at 1732036600006Finished memstore snapshotting TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732036600006Flushing stores of TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb. at 1732036600006Flushing a6bc673cb3ab3290d1b783693a45a9bb/A: creating writer at 1732036600006Flushing mob file a6bc673cb3ab3290d1b783693a45a9bb/A: appending metadata at 1732036600010 (+4 ms)Flushing mob file a6bc673cb3ab3290d1b783693a45a9bb/A: closing flushed file at 1732036600010Flushing a6bc673cb3ab3290d1b783693a45a9bb/A: appending metadata at 1732036600417 (+407 ms)Flushing a6bc673cb3ab3290d1b783693a45a9bb/A: closing flushed file at 1732036600417Flushing a6bc673cb3ab3290d1b783693a45a9bb/B: creating writer at 1732036600824 (+407 ms)Flushing a6bc673cb3ab3290d1b783693a45a9bb/B: appending metadata at 1732036600826 (+2 ms)Flushing a6bc673cb3ab3290d1b783693a45a9bb/B: closing flushed file at 1732036600826Flushing a6bc673cb3ab3290d1b783693a45a9bb/C: creating writer at 1732036601232 (+406 ms)Flushing a6bc673cb3ab3290d1b783693a45a9bb/C: appending metadata at 1732036601234 (+2 ms)Flushing a6bc673cb3ab3290d1b783693a45a9bb/C: closing flushed file at 1732036601234Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@573f9b3e: reopening flushed file at 1732036601640 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6fa4428b: reopening flushed file at 1732036601644 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ed3dcf4: reopening flushed file at 1732036601647 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for a6bc673cb3ab3290d1b783693a45a9bb in 1645ms, sequenceid=514, compaction requested=true at 1732036601650 (+3 ms)Writing region close event to WAL at 1732036601706 (+56 ms)Running coprocessor post-close hooks at 1732036601709 (+3 ms)Closed at 1732036601709 2024-11-19T17:16:41,711 INFO [RS_CLOSE_REGION-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] handler.UnassignRegionHandler(157): Closed a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:41,711 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=182 updating hbase:meta row=a6bc673cb3ab3290d1b783693a45a9bb, regionState=CLOSED 2024-11-19T17:16:41,712 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=183, ppid=182, state=RUNNABLE, hasLock=false; CloseRegionProcedure a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 because future has completed 2024-11-19T17:16:41,714 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=183, resume processing ppid=182 2024-11-19T17:16:41,715 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=183, ppid=182, state=SUCCESS, hasLock=false; CloseRegionProcedure a6bc673cb3ab3290d1b783693a45a9bb, server=280a42d09ec8,33873,1732036422473 in 1.8600 sec 2024-11-19T17:16:41,716 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=182, resume processing ppid=181 2024-11-19T17:16:41,716 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=182, ppid=181, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6bc673cb3ab3290d1b783693a45a9bb, UNASSIGN in 1.8630 sec 2024-11-19T17:16:41,717 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=181, resume processing ppid=180 2024-11-19T17:16:41,717 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=181, ppid=180, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8660 sec 2024-11-19T17:16:41,718 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732036601718"}]},"ts":"1732036601718"} 2024-11-19T17:16:41,719 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T17:16:41,719 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T17:16:41,721 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=180, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8740 sec 2024-11-19T17:16:41,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-19T17:16:41,994 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:16:41,995 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T17:16:41,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] procedure2.ProcedureExecutor(1139): Stored pid=184, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:16:41,997 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=184, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:16:41,998 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=184, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:16:41,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=184 2024-11-19T17:16:42,000 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:42,002 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A, FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B, FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C, FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/recovered.edits] 2024-11-19T17:16:42,005 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/641144a42ddf4fd29df1e2d19929f252 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/641144a42ddf4fd29df1e2d19929f252 2024-11-19T17:16:42,008 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/7a803bf4ca6047dda56f37a1518dd218 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/7a803bf4ca6047dda56f37a1518dd218 2024-11-19T17:16:42,009 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/90e1ee873c624f69a8ad1e0d923b68ff to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/A/90e1ee873c624f69a8ad1e0d923b68ff 2024-11-19T17:16:42,011 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/49df549a3aba4d97ae424192c1800bdd to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/49df549a3aba4d97ae424192c1800bdd 2024-11-19T17:16:42,013 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/ae19d5cb9e4a4e8598ad96b32f1158ed to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/ae19d5cb9e4a4e8598ad96b32f1158ed 2024-11-19T17:16:42,014 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/e96584f6b9234158905d9dc22729c926 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/B/e96584f6b9234158905d9dc22729c926 2024-11-19T17:16:42,016 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/67c72acf8d37444c9d2d30c4b7e70a4f to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/67c72acf8d37444c9d2d30c4b7e70a4f 2024-11-19T17:16:42,017 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/7d21df0a894e4a10b5409045703912de to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/7d21df0a894e4a10b5409045703912de 2024-11-19T17:16:42,018 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/b89ec53b51ca4a98b60e5418de53e889 to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/C/b89ec53b51ca4a98b60e5418de53e889 2024-11-19T17:16:42,021 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/recovered.edits/517.seqid to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb/recovered.edits/517.seqid 2024-11-19T17:16:42,021 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/default/TestAcidGuarantees/a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:42,021 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T17:16:42,022 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T17:16:42,022 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T17:16:42,025 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111901ca7300b8de44a3adbce7e5d51ac236_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111901ca7300b8de44a3adbce7e5d51ac236_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:42,026 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190827a01bbcd44522bbb380ffdb7968f5_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190827a01bbcd44522bbb380ffdb7968f5_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:42,028 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119203e2c03315b4adb9111adfd29b306d1_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119203e2c03315b4adb9111adfd29b306d1_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:42,029 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192aa1004e407044ffb73d2f691bfaf0dc_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192aa1004e407044ffb73d2f691bfaf0dc_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:42,030 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119378811244e244188b7692c9d049ccd41_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119378811244e244188b7692c9d049ccd41_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:42,031 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195793fcafed3740c5996d01dca44c13b5_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195793fcafed3740c5996d01dca44c13b5_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:42,032 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119659e34c71fd34f329d40eac1531cd518_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119659e34c71fd34f329d40eac1531cd518_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:42,033 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119713bd99daa9443209c789f2bbf57a75d_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119713bd99daa9443209c789f2bbf57a75d_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:42,034 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197e479f695ebe4fe6adaa534ebfa24fdb_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197e479f695ebe4fe6adaa534ebfa24fdb_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:42,036 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198abb66ecabf54dc5ba8f41a29b619568_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198abb66ecabf54dc5ba8f41a29b619568_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:42,038 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198cb8c677a0ea4c99b3be6bca8280a983_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198cb8c677a0ea4c99b3be6bca8280a983_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:42,039 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ae625b75aa114b63b4a5e70fc933787e_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ae625b75aa114b63b4a5e70fc933787e_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:42,040 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c0605575761a42efaab5c1f4e32b06a4_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c0605575761a42efaab5c1f4e32b06a4_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:42,041 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c121dcac99eb44a1bc757d618982d2d2_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c121dcac99eb44a1bc757d618982d2d2_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:42,042 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c2c00e5c5cc946c79ff1e2c5dfc6de83_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c2c00e5c5cc946c79ff1e2c5dfc6de83_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:42,043 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d1001dd414a4423bbf8fc4598261944b_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d1001dd414a4423bbf8fc4598261944b_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:42,044 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e602718d14be489e8de3d9b9edfeb3d7_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e602718d14be489e8de3d9b9edfeb3d7_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:42,045 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ebd549b6998e41aca98298bc11ee9add_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ebd549b6998e41aca98298bc11ee9add_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:42,046 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f3062c73db2943d1a7a57ace57e27d93_a6bc673cb3ab3290d1b783693a45a9bb to hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f3062c73db2943d1a7a57ace57e27d93_a6bc673cb3ab3290d1b783693a45a9bb 2024-11-19T17:16:42,047 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T17:16:42,050 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=184, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:16:42,053 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T17:16:42,055 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T17:16:42,057 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=184, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:16:42,057 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T17:16:42,057 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732036602057"}]},"ts":"9223372036854775807"} 2024-11-19T17:16:42,059 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T17:16:42,059 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a6bc673cb3ab3290d1b783693a45a9bb, NAME => 'TestAcidGuarantees,,1732036567032.a6bc673cb3ab3290d1b783693a45a9bb.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T17:16:42,059 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T17:16:42,059 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732036602059"}]},"ts":"9223372036854775807"} 2024-11-19T17:16:42,061 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T17:16:42,061 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=184, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T17:16:42,062 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=184, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 66 msec 2024-11-19T17:16:42,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=184 2024-11-19T17:16:42,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T17:16:42,104 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T17:16:42,116 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 250), OpenFileDescriptor=445 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=301 (was 366), ProcessCount=11 (was 11), AvailableMemoryMB=5985 (was 6096) 2024-11-19T17:16:42,116 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-19T17:16:42,116 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T17:16:42,116 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T17:16:42,117 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:42,117 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:42,117 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:16:42,117 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-19T17:16:42,117 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=841754823, stopped=false 2024-11-19T17:16:42,117 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=280a42d09ec8,40603,1732036421715 2024-11-19T17:16:42,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33873-0x10131384f3a0001, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T17:16:42,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x10131384f3a0000, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T17:16:42,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x10131384f3a0000, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:16:42,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33873-0x10131384f3a0001, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:16:42,119 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T17:16:42,120 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33873-0x10131384f3a0001, quorum=127.0.0.1:56338, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T17:16:42,120 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T17:16:42,120 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40603-0x10131384f3a0000, quorum=127.0.0.1:56338, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T17:16:42,120 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T17:16:42,120 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:42,121 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '280a42d09ec8,33873,1732036422473' ***** 2024-11-19T17:16:42,121 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-19T17:16:42,121 INFO [RS:0;280a42d09ec8:33873 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T17:16:42,121 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T17:16:42,121 INFO [RS:0;280a42d09ec8:33873 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T17:16:42,122 INFO [RS:0;280a42d09ec8:33873 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T17:16:42,122 INFO [RS:0;280a42d09ec8:33873 {}] regionserver.HRegionServer(959): stopping server 280a42d09ec8,33873,1732036422473 2024-11-19T17:16:42,122 INFO [RS:0;280a42d09ec8:33873 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T17:16:42,122 INFO [RS:0;280a42d09ec8:33873 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;280a42d09ec8:33873. 2024-11-19T17:16:42,122 DEBUG [RS:0;280a42d09ec8:33873 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T17:16:42,122 DEBUG [RS:0;280a42d09ec8:33873 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:42,122 INFO [RS:0;280a42d09ec8:33873 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T17:16:42,122 INFO [RS:0;280a42d09ec8:33873 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T17:16:42,122 INFO [RS:0;280a42d09ec8:33873 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T17:16:42,122 INFO [RS:0;280a42d09ec8:33873 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-19T17:16:42,123 INFO [RS:0;280a42d09ec8:33873 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-19T17:16:42,123 DEBUG [RS:0;280a42d09ec8:33873 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-19T17:16:42,123 DEBUG [RS:0;280a42d09ec8:33873 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T17:16:42,123 DEBUG [RS_CLOSE_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T17:16:42,123 INFO [RS_CLOSE_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T17:16:42,123 DEBUG [RS_CLOSE_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T17:16:42,123 DEBUG [RS_CLOSE_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T17:16:42,123 DEBUG [RS_CLOSE_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T17:16:42,123 INFO [RS_CLOSE_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-19T17:16:42,132 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T17:16:42,138 INFO [regionserver/280a42d09ec8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T17:16:42,146 DEBUG [RS_CLOSE_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/hbase/meta/1588230740/.tmp/info/71dcbf2ed205489293bc54232d602bb5 is 95, key is TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3./info:/1732036453103/DeleteFamily/seqid=0 2024-11-19T17:16:42,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742523_1699 (size=6514) 2024-11-19T17:16:42,253 INFO [regionserver/280a42d09ec8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-19T17:16:42,253 INFO [regionserver/280a42d09ec8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-19T17:16:42,323 DEBUG [RS:0;280a42d09ec8:33873 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T17:16:42,524 DEBUG [RS:0;280a42d09ec8:33873 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T17:16:42,550 INFO [RS_CLOSE_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/hbase/meta/1588230740/.tmp/info/71dcbf2ed205489293bc54232d602bb5 2024-11-19T17:16:42,567 DEBUG [RS_CLOSE_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/hbase/meta/1588230740/.tmp/ns/86e348da5a944a4cb2f6c34ffdf7ffda is 93, key is TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3./ns:/1732036453049/DeleteFamily/seqid=0 2024-11-19T17:16:42,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742524_1700 (size=5906) 2024-11-19T17:16:42,724 DEBUG [RS:0;280a42d09ec8:33873 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T17:16:42,924 DEBUG [RS:0;280a42d09ec8:33873 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T17:16:42,971 INFO [RS_CLOSE_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/hbase/meta/1588230740/.tmp/ns/86e348da5a944a4cb2f6c34ffdf7ffda 2024-11-19T17:16:42,989 DEBUG [RS_CLOSE_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/hbase/meta/1588230740/.tmp/rep_barrier/c8fe3292b5224440b59af6149e9972c5 is 102, key is TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3./rep_barrier:/1732036453049/DeleteFamily/seqid=0 2024-11-19T17:16:42,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742525_1701 (size=6025) 2024-11-19T17:16:43,124 DEBUG [RS:0;280a42d09ec8:33873 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T17:16:43,324 DEBUG [RS:0;280a42d09ec8:33873 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T17:16:43,393 INFO [RS_CLOSE_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/hbase/meta/1588230740/.tmp/rep_barrier/c8fe3292b5224440b59af6149e9972c5 2024-11-19T17:16:43,410 DEBUG [RS_CLOSE_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/hbase/meta/1588230740/.tmp/table/cdfa5f69ac4f49248751f0022c5c0a66 is 96, key is TestAcidGuarantees,,1732036426462.689491498ff99efc87002cc59bc70ef3./table:/1732036453049/DeleteFamily/seqid=0 2024-11-19T17:16:43,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742526_1702 (size=5936) 2024-11-19T17:16:43,525 DEBUG [RS:0;280a42d09ec8:33873 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T17:16:43,725 DEBUG [RS:0;280a42d09ec8:33873 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T17:16:43,814 INFO [RS_CLOSE_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/hbase/meta/1588230740/.tmp/table/cdfa5f69ac4f49248751f0022c5c0a66 2024-11-19T17:16:43,817 DEBUG [RS_CLOSE_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/hbase/meta/1588230740/.tmp/info/71dcbf2ed205489293bc54232d602bb5 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/hbase/meta/1588230740/info/71dcbf2ed205489293bc54232d602bb5 2024-11-19T17:16:43,819 INFO [RS_CLOSE_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/hbase/meta/1588230740/info/71dcbf2ed205489293bc54232d602bb5, entries=12, sequenceid=90, filesize=6.4 K 2024-11-19T17:16:43,820 DEBUG [RS_CLOSE_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/hbase/meta/1588230740/.tmp/ns/86e348da5a944a4cb2f6c34ffdf7ffda as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/hbase/meta/1588230740/ns/86e348da5a944a4cb2f6c34ffdf7ffda 2024-11-19T17:16:43,823 INFO [RS_CLOSE_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/hbase/meta/1588230740/ns/86e348da5a944a4cb2f6c34ffdf7ffda, entries=8, sequenceid=90, filesize=5.8 K 2024-11-19T17:16:43,823 DEBUG [RS_CLOSE_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/hbase/meta/1588230740/.tmp/rep_barrier/c8fe3292b5224440b59af6149e9972c5 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/hbase/meta/1588230740/rep_barrier/c8fe3292b5224440b59af6149e9972c5 2024-11-19T17:16:43,826 INFO [RS_CLOSE_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/hbase/meta/1588230740/rep_barrier/c8fe3292b5224440b59af6149e9972c5, entries=6, sequenceid=90, filesize=5.9 K 2024-11-19T17:16:43,826 DEBUG [RS_CLOSE_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/hbase/meta/1588230740/.tmp/table/cdfa5f69ac4f49248751f0022c5c0a66 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/hbase/meta/1588230740/table/cdfa5f69ac4f49248751f0022c5c0a66 2024-11-19T17:16:43,828 INFO [RS_CLOSE_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/hbase/meta/1588230740/table/cdfa5f69ac4f49248751f0022c5c0a66, entries=7, sequenceid=90, filesize=5.8 K 2024-11-19T17:16:43,829 INFO [RS_CLOSE_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1706ms, sequenceid=90, compaction requested=false 2024-11-19T17:16:43,833 DEBUG [RS_CLOSE_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-19T17:16:43,833 DEBUG [RS_CLOSE_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T17:16:43,833 INFO [RS_CLOSE_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T17:16:43,834 DEBUG [RS_CLOSE_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732036602123Running coprocessor pre-close hooks at 1732036602123Disabling compacts and flushes for region at 1732036602123Disabling writes for close at 1732036602123Obtaining lock to block concurrent updates at 1732036602123Preparing flush snapshotting stores in 1588230740 at 1732036602123Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732036602124 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732036602124Flushing 1588230740/info: creating writer at 1732036602125 (+1 ms)Flushing 1588230740/info: appending metadata at 1732036602146 (+21 ms)Flushing 1588230740/info: closing flushed file at 1732036602146Flushing 1588230740/ns: creating writer at 1732036602553 (+407 ms)Flushing 1588230740/ns: appending metadata at 1732036602567 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732036602567Flushing 1588230740/rep_barrier: creating writer at 1732036602975 (+408 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732036602989 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732036602989Flushing 1588230740/table: creating writer at 1732036603396 (+407 ms)Flushing 1588230740/table: appending metadata at 1732036603410 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732036603410Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c959a89: reopening flushed file at 1732036603817 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@547058eb: reopening flushed file at 1732036603820 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@291999bb: reopening flushed file at 1732036603823 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@afd9119: reopening flushed file at 1732036603826 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1706ms, sequenceid=90, compaction requested=false at 1732036603829 (+3 ms)Writing region close event to WAL at 1732036603831 (+2 ms)Running coprocessor post-close hooks at 1732036603833 (+2 ms)Closed at 1732036603833 2024-11-19T17:16:43,834 DEBUG [RS_CLOSE_META-regionserver/280a42d09ec8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-19T17:16:43,925 INFO [RS:0;280a42d09ec8:33873 {}] regionserver.HRegionServer(976): stopping server 280a42d09ec8,33873,1732036422473; all regions closed. 2024-11-19T17:16:43,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741834_1010 (size=25344) 2024-11-19T17:16:43,931 DEBUG [RS:0;280a42d09ec8:33873 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/oldWALs 2024-11-19T17:16:43,931 INFO [RS:0;280a42d09ec8:33873 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 280a42d09ec8%2C33873%2C1732036422473.meta:.meta(num 1732036426024) 2024-11-19T17:16:43,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741832_1008 (size=20314477) 2024-11-19T17:16:43,935 DEBUG [RS:0;280a42d09ec8:33873 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/oldWALs 2024-11-19T17:16:43,935 INFO [RS:0;280a42d09ec8:33873 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 280a42d09ec8%2C33873%2C1732036422473:(num 1732036425418) 2024-11-19T17:16:43,935 DEBUG [RS:0;280a42d09ec8:33873 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:16:43,935 INFO [RS:0;280a42d09ec8:33873 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T17:16:43,935 INFO [RS:0;280a42d09ec8:33873 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T17:16:43,935 INFO [RS:0;280a42d09ec8:33873 {}] hbase.ChoreService(370): Chore service for: regionserver/280a42d09ec8:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-19T17:16:43,935 INFO [RS:0;280a42d09ec8:33873 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T17:16:43,935 INFO [regionserver/280a42d09ec8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T17:16:43,936 INFO [RS:0;280a42d09ec8:33873 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33873 2024-11-19T17:16:43,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33873-0x10131384f3a0001, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/280a42d09ec8,33873,1732036422473 2024-11-19T17:16:43,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x10131384f3a0000, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T17:16:43,940 INFO [RS:0;280a42d09ec8:33873 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T17:16:43,942 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [280a42d09ec8,33873,1732036422473] 2024-11-19T17:16:43,945 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/280a42d09ec8,33873,1732036422473 already deleted, retry=false 2024-11-19T17:16:43,945 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 280a42d09ec8,33873,1732036422473 expired; onlineServers=0 2024-11-19T17:16:43,945 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '280a42d09ec8,40603,1732036421715' ***** 2024-11-19T17:16:43,945 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-19T17:16:43,945 INFO [M:0;280a42d09ec8:40603 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T17:16:43,945 INFO [M:0;280a42d09ec8:40603 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T17:16:43,945 DEBUG [M:0;280a42d09ec8:40603 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-19T17:16:43,946 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-19T17:16:43,946 DEBUG [M:0;280a42d09ec8:40603 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-19T17:16:43,946 DEBUG [master/280a42d09ec8:0:becomeActiveMaster-HFileCleaner.large.0-1732036425123 {}] cleaner.HFileCleaner(306): Exit Thread[master/280a42d09ec8:0:becomeActiveMaster-HFileCleaner.large.0-1732036425123,5,FailOnTimeoutGroup] 2024-11-19T17:16:43,946 DEBUG [master/280a42d09ec8:0:becomeActiveMaster-HFileCleaner.small.0-1732036425124 {}] cleaner.HFileCleaner(306): Exit Thread[master/280a42d09ec8:0:becomeActiveMaster-HFileCleaner.small.0-1732036425124,5,FailOnTimeoutGroup] 2024-11-19T17:16:43,946 INFO [M:0;280a42d09ec8:40603 {}] hbase.ChoreService(370): Chore service for: master/280a42d09ec8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-19T17:16:43,946 INFO [M:0;280a42d09ec8:40603 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T17:16:43,946 DEBUG [M:0;280a42d09ec8:40603 {}] master.HMaster(1795): Stopping service threads 2024-11-19T17:16:43,946 INFO [M:0;280a42d09ec8:40603 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-19T17:16:43,946 INFO [M:0;280a42d09ec8:40603 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T17:16:43,946 ERROR [M:0;280a42d09ec8:40603 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-19T17:16:43,947 INFO [M:0;280a42d09ec8:40603 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T17:16:43,947 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-19T17:16:43,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x10131384f3a0000, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-19T17:16:43,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x10131384f3a0000, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:16:43,948 DEBUG [M:0;280a42d09ec8:40603 {}] zookeeper.ZKUtil(347): master:40603-0x10131384f3a0000, quorum=127.0.0.1:56338, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-19T17:16:43,948 WARN [M:0;280a42d09ec8:40603 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-19T17:16:43,948 INFO [M:0;280a42d09ec8:40603 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/.lastflushedseqids 2024-11-19T17:16:43,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742527_1703 (size=72) 2024-11-19T17:16:44,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33873-0x10131384f3a0001, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T17:16:44,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33873-0x10131384f3a0001, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T17:16:44,043 INFO [RS:0;280a42d09ec8:33873 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T17:16:44,043 INFO [RS:0;280a42d09ec8:33873 {}] regionserver.HRegionServer(1031): Exiting; stopping=280a42d09ec8,33873,1732036422473; zookeeper connection closed. 2024-11-19T17:16:44,044 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@55370f6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@55370f6 2024-11-19T17:16:44,044 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-19T17:16:44,357 INFO [M:0;280a42d09ec8:40603 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T17:16:44,358 INFO [M:0;280a42d09ec8:40603 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-19T17:16:44,358 DEBUG [M:0;280a42d09ec8:40603 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T17:16:44,358 INFO [M:0;280a42d09ec8:40603 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:16:44,358 DEBUG [M:0;280a42d09ec8:40603 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:16:44,358 DEBUG [M:0;280a42d09ec8:40603 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T17:16:44,358 DEBUG [M:0;280a42d09ec8:40603 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:16:44,358 INFO [M:0;280a42d09ec8:40603 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=810.55 KB heapSize=995.31 KB 2024-11-19T17:16:44,373 DEBUG [M:0;280a42d09ec8:40603 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fe1ba370477941bb8bec05339daa968c is 82, key is hbase:meta,,1/info:regioninfo/1732036426127/Put/seqid=0 2024-11-19T17:16:44,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742528_1704 (size=5672) 2024-11-19T17:16:44,776 INFO [M:0;280a42d09ec8:40603 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2280 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fe1ba370477941bb8bec05339daa968c 2024-11-19T17:16:44,796 DEBUG [M:0;280a42d09ec8:40603 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9b084e1e437e499fb08762624bde81ff is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x9B/proc:d/1732036570102/Put/seqid=0 2024-11-19T17:16:44,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742529_1705 (size=47790) 2024-11-19T17:16:45,200 INFO [M:0;280a42d09ec8:40603 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=809.99 KB at sequenceid=2280 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9b084e1e437e499fb08762624bde81ff 2024-11-19T17:16:45,203 INFO [M:0;280a42d09ec8:40603 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9b084e1e437e499fb08762624bde81ff 2024-11-19T17:16:45,219 DEBUG [M:0;280a42d09ec8:40603 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fc3cd9f7dc6343adb6906a70d06647a6 is 69, key is 280a42d09ec8,33873,1732036422473/rs:state/1732036425172/Put/seqid=0 2024-11-19T17:16:45,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073742530_1706 (size=5156) 2024-11-19T17:16:45,623 INFO [M:0;280a42d09ec8:40603 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2280 (bloomFilter=true), to=hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fc3cd9f7dc6343adb6906a70d06647a6 2024-11-19T17:16:45,626 DEBUG [M:0;280a42d09ec8:40603 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fe1ba370477941bb8bec05339daa968c as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fe1ba370477941bb8bec05339daa968c 2024-11-19T17:16:45,628 INFO [M:0;280a42d09ec8:40603 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fe1ba370477941bb8bec05339daa968c, entries=8, sequenceid=2280, filesize=5.5 K 2024-11-19T17:16:45,629 DEBUG [M:0;280a42d09ec8:40603 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9b084e1e437e499fb08762624bde81ff as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9b084e1e437e499fb08762624bde81ff 2024-11-19T17:16:45,631 INFO [M:0;280a42d09ec8:40603 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9b084e1e437e499fb08762624bde81ff 2024-11-19T17:16:45,631 INFO [M:0;280a42d09ec8:40603 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9b084e1e437e499fb08762624bde81ff, entries=184, sequenceid=2280, filesize=46.7 K 2024-11-19T17:16:45,632 DEBUG [M:0;280a42d09ec8:40603 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fc3cd9f7dc6343adb6906a70d06647a6 as hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fc3cd9f7dc6343adb6906a70d06647a6 2024-11-19T17:16:45,634 INFO [M:0;280a42d09ec8:40603 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34575/user/jenkins/test-data/7cf40541-f6ec-3c6d-7e97-38eedc0f009e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fc3cd9f7dc6343adb6906a70d06647a6, entries=1, sequenceid=2280, filesize=5.0 K 2024-11-19T17:16:45,634 INFO [M:0;280a42d09ec8:40603 {}] regionserver.HRegion(3140): Finished flush of dataSize ~810.55 KB/830002, heapSize ~995.02 KB/1018896, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1276ms, sequenceid=2280, compaction requested=false 2024-11-19T17:16:45,636 INFO [M:0;280a42d09ec8:40603 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:16:45,636 DEBUG [M:0;280a42d09ec8:40603 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732036604358Disabling compacts and flushes for region at 1732036604358Disabling writes for close at 1732036604358Obtaining lock to block concurrent updates at 1732036604358Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732036604358Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=830002, getHeapSize=1019136, getOffHeapSize=0, getCellsCount=2489 at 1732036604358Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732036604359 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732036604359Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732036604373 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732036604373Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732036604780 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732036604796 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732036604796Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732036605203 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732036605219 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732036605219Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60b7a8e7: reopening flushed file at 1732036605625 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5262b171: reopening flushed file at 1732036605628 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d5a147f: reopening flushed file at 1732036605631 (+3 ms)Finished flush of dataSize ~810.55 KB/830002, heapSize ~995.02 KB/1018896, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1276ms, sequenceid=2280, compaction requested=false at 1732036605634 (+3 ms)Writing region close event to WAL at 1732036605636 (+2 ms)Closed at 1732036605636 2024-11-19T17:16:45,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43609 is added to blk_1073741830_1006 (size=979457) 2024-11-19T17:16:45,638 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T17:16:45,639 INFO [M:0;280a42d09ec8:40603 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-19T17:16:45,639 INFO [M:0;280a42d09ec8:40603 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40603 2024-11-19T17:16:45,639 INFO [M:0;280a42d09ec8:40603 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T17:16:45,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x10131384f3a0000, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T17:16:45,741 INFO [M:0;280a42d09ec8:40603 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T17:16:45,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40603-0x10131384f3a0000, quorum=127.0.0.1:56338, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T17:16:45,746 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T17:16:45,748 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T17:16:45,748 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T17:16:45,749 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-19T17:16:45,749 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/hadoop.log.dir/,STOPPED} 2024-11-19T17:16:45,752 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-19T17:16:45,752 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-19T17:16:45,752 WARN [BP-1636120753-172.17.0.2-1732036416839 heartbeating to localhost/127.0.0.1:34575 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-19T17:16:45,752 WARN [BP-1636120753-172.17.0.2-1732036416839 heartbeating to localhost/127.0.0.1:34575 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1636120753-172.17.0.2-1732036416839 (Datanode Uuid 040f1250-2e83-45c5-ad64-23874f827109) service to localhost/127.0.0.1:34575 2024-11-19T17:16:45,755 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/cluster_8102c339-c42d-0a52-7e0c-92f0478cebda/data/data1/current/BP-1636120753-172.17.0.2-1732036416839 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T17:16:45,755 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/cluster_8102c339-c42d-0a52-7e0c-92f0478cebda/data/data2/current/BP-1636120753-172.17.0.2-1732036416839 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T17:16:45,756 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-19T17:16:45,769 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-19T17:16:45,770 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T17:16:45,770 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T17:16:45,770 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-19T17:16:45,770 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0aab3372-d41e-c3e6-10fa-e39ab0ef125e/hadoop.log.dir/,STOPPED} 2024-11-19T17:16:45,787 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-19T17:16:45,924 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down